Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions apps/worker/scripts/debug_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ def _finalize_output(
logger.info(" 📊 POST-PARSE TIMELINE")
logger.info("═" * 58)
for phase, elapsed in timings.items():
pct = (elapsed / t_total * 100) if t_total > 0 else 0
pct = elapsed / t_total * 100
logger.info(f" {phase:<35s} │ {elapsed:>7.2f}s ({pct:>5.1f}%)")
logger.info(" " + "─" * 55)
logger.info(f" {'TOTAL':<35s} │ {t_total:>7.2f}s (100.0%)")
Expand Down Expand Up @@ -487,15 +487,13 @@ def run_pipeline(
init_token_tracker,
)

token_usage_dict = get_current_token_tracker()
owns_token_tracker = token_usage_dict is None
if token_usage_dict is None:
token_usage_dict = init_token_tracker()
owns_token_tracker = get_current_token_tracker() is None
if owns_token_tracker:
init_token_tracker()

stage_timing_dict = get_current_stage_tracker()
owns_stage_tracker = stage_timing_dict is None
if stage_timing_dict is None:
stage_timing_dict = init_stage_tracker()
owns_stage_tracker = get_current_stage_tracker() is None
if owns_stage_tracker:
init_stage_tracker()

try:
if output_root_override:
Expand Down
1 change: 0 additions & 1 deletion apps/worker/scripts/debug_retrieval.py
Original file line number Diff line number Diff line change
Expand Up @@ -1100,7 +1100,6 @@ def _render_md_report(all_reports: list[dict[str, Any]]) -> str:


async def main() -> None:
import re
from datetime import datetime

# Enable verbose logging to see full LLM prompts and responses
Expand Down
36 changes: 16 additions & 20 deletions apps/worker/scripts/page_memory/_debug_pm_shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,17 @@
)
from shared.services.ai.token_costing import build_token_cost_estimate

# Re-exported for staged debug scripts. Listing them here marks the imports as
# intentional so CodeQL does not treat them as unused.
__all__ = [
"page_scope_info",
"scope_id_for_pages",
"_build_hierarchy_from_skeletons",
"_derive_hierarchy_page_scope",
"_scope_manifest",
"_serialize_scope_skeletons",
]

DEFAULT_PDF = Path(
"/Users/wuchengke/Desktop/temp/test_docs/"
"SJSYJ-SC-2024 企业制度汇编(上册).pdf"
Expand Down Expand Up @@ -866,11 +877,7 @@ def write_toc_hierarchy_artifact(
path = out_dir / "toc_hierarchy.json"
# Legacy list dump from an earlier debug format.
for legacy_name in ("toc_hierarchies.json",):
legacy = out_dir / legacy_name
try:
legacy.unlink()
except FileNotFoundError:
pass
(out_dir / legacy_name).unlink(missing_ok=True)
write_debug_json(
path,
{
Expand Down Expand Up @@ -1046,19 +1053,13 @@ def remove_legacy_doc_agent_artifacts(
}
)
for name in names:
try:
(doc_agent_dir / name).unlink()
except FileNotFoundError:
pass
(doc_agent_dir / name).unlink(missing_ok=True)
legacy_preview_dir = doc_agent_dir / "coarse_assets"
if legacy_preview_dir.is_dir():
import shutil

shutil.rmtree(legacy_preview_dir)
try:
(doc_agent_dir / "coarse_assets.html").unlink()
except FileNotFoundError:
pass
(doc_agent_dir / "coarse_assets.html").unlink(missing_ok=True)


def record_stage(
Expand Down Expand Up @@ -1399,9 +1400,7 @@ def stop_with_trace(

def remove_nested_doc_agent_trace(out_dir: Path) -> None:
try:
(out_dir / "_doc_agent" / "trace.json").unlink()
except FileNotFoundError:
pass
(out_dir / "_doc_agent" / "trace.json").unlink(missing_ok=True)
except Exception as exc:
logger.debug(f"failed to remove nested doc-agent trace: {exc}")

Expand Down Expand Up @@ -1452,10 +1451,7 @@ def write_top_level_artifacts(
if assets_by_page is not None:
write_debug_json(out_dir / "assets.json", _serialize_assets(assets_by_page))
else:
try:
(out_dir / "assets.json").unlink()
except FileNotFoundError:
pass
(out_dir / "assets.json").unlink(missing_ok=True)


def cleanup_page_memory_artifacts(out_dir: Path) -> None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
import time

from loguru import logger
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
import time

from loguru import logger
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
from loguru import logger

from _debug_pm_shared import (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
import time

from loguru import logger
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
import os
import time
from pathlib import Path
Expand Down
1 change: 0 additions & 1 deletion apps/worker/scripts/page_memory/debug_pm_stage5_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
import os
import time
from dataclasses import dataclass
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

sys.path.insert(0, str(_Path(__file__).resolve().parent))

from _debug_pm_shared import * # noqa: F401,F403
import json
import os
import time
Expand Down Expand Up @@ -103,8 +102,12 @@ def _run_tagging_for_scope(
if assets_path.exists():
try:
assets_by_page = load_assets_artifact(assets_path)
except Exception:
pass
except Exception as exc:
logger.warning(
" failed to load {}; continuing without assets: {}",
assets_path,
exc,
)

# Reuse Stage-4's exact scope contract. Fall back only for old artifacts.
recorded_pages = prior_scope.get("processing_pages")
Expand Down
Loading