diff --git a/apps/api/.env.example b/apps/api/.env.example index 8ab1c0aac..ef00d0ff0 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -110,20 +110,8 @@ MAX_FILE_SIZE=314572800 MAX_PDF_PAGE_LIMIT=200 OVERSIZED_PDF_SHARD_ENABLED=true OVERSIZED_PDF_SOFT_LIMIT=1500 -PDF_PROFILE_TOC_ENABLED=false +PDF_PROFILE_TOC_ENABLED=true MINERU_SHARD_CONCURRENCY=3 -PARSE_AGENT_PLAN_BUDGET=50000 -PARSE_AGENT_VISUAL_BUDGET=120000 -PARSE_AGENT_TOC_CONFIRM_MIN_BUDGET=8000 -PARSE_AGENT_TOC_CONFIRM_CAP=24000 -PARSE_AGENT_COARSE_PLANNER_MIN_BUDGET=12000 -PARSE_AGENT_COARSE_PLANNER_CAP=36000 -PARSE_AGENT_STRUCTURAL_REACT_MIN_BUDGET=24000 -PARSE_AGENT_STRUCTURAL_REACT_CAP=64000 -PARSE_AGENT_CALIBRATION_MIN_BUDGET=12000 -PARSE_AGENT_CALIBRATION_CAP=40000 -PARSE_AGENT_PAGE_TAGGING_MIN_BUDGET=0 -PARSE_AGENT_PAGE_TAGGING_CAP=0 # Required for specific features: webhooks and callbacks WEBHOOK_MASTER_KEY= diff --git a/apps/worker/.env.example b/apps/worker/.env.example index 5e4d118e7..46a347c88 100644 --- a/apps/worker/.env.example +++ b/apps/worker/.env.example @@ -125,22 +125,10 @@ MAX_FILE_SIZE=314572800 MAX_PDF_PAGE_LIMIT=200 OVERSIZED_PDF_SHARD_ENABLED=true OVERSIZED_PDF_SOFT_LIMIT=1500 -PDF_PROFILE_TOC_ENABLED=false +PDF_PROFILE_TOC_ENABLED=true # macOS/Homebrew local debugging only; worker Docker image installs OpenJDK. # JAVA_HOME=/opt/homebrew/opt/java MINERU_SHARD_CONCURRENCY=3 -PARSE_AGENT_PLAN_BUDGET=50000 -PARSE_AGENT_VISUAL_BUDGET=120000 -PARSE_AGENT_TOC_CONFIRM_MIN_BUDGET=8000 -PARSE_AGENT_TOC_CONFIRM_CAP=24000 -PARSE_AGENT_COARSE_PLANNER_MIN_BUDGET=12000 -PARSE_AGENT_COARSE_PLANNER_CAP=36000 -PARSE_AGENT_STRUCTURAL_REACT_MIN_BUDGET=24000 -PARSE_AGENT_STRUCTURAL_REACT_CAP=64000 -PARSE_AGENT_CALIBRATION_MIN_BUDGET=12000 -PARSE_AGENT_CALIBRATION_CAP=40000 -PARSE_AGENT_PAGE_TAGGING_MIN_BUDGET=0 -PARSE_AGENT_PAGE_TAGGING_CAP=0 # Parser row schema. `entities` (JSON typed entities, §4.4) and `asset_title` # (asset caption/label, §4.5) are additive trailing columns; the parser layer diff --git a/apps/worker/app/services/document_agent/__init__.py b/apps/worker/app/services/document_agent/__init__.py index 6baae446c..500505013 100644 --- a/apps/worker/app/services/document_agent/__init__.py +++ b/apps/worker/app/services/document_agent/__init__.py @@ -1,4 +1,4 @@ -"""Page anatomy agent for hierarchy-first PDF profiling.""" +"""Document profile workflow for hierarchy-first PDF profiling.""" from app.services.document_agent.manifest import ( PageAnatomyMap, diff --git a/apps/worker/app/services/document_agent/agents/__init__.py b/apps/worker/app/services/document_agent/agents/__init__.py deleted file mode 100644 index 9cd87f956..000000000 --- a/apps/worker/app/services/document_agent/agents/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Calibration agents package.""" diff --git a/apps/worker/app/services/document_agent/agents/calibration/SKILL.md b/apps/worker/app/services/document_agent/agents/calibration/SKILL.md deleted file mode 100644 index cc3c3f932..000000000 --- a/apps/worker/app/services/document_agent/agents/calibration/SKILL.md +++ /dev/null @@ -1,86 +0,0 @@ -# Calibration SubAgent Skill - -## Goal - -For the **current TOC region**, discover page-numbering **regimes** and an -**initial offset** for each regime that has usable entries. Submit candidate -offsets via `calibration.submit`. After submit, a deterministic completion pass -runs the production tail-verify → binary-search → small-step recalibrate loop -(using the same visual page confirmer as production). Only **complete segments** -are usable for coarse structure; unrecognized pages are treated as **no TOC**. - -## Do not use - -- Do not scan a fixed window after the TOC (the old “TOC end + N pages” probe). -- Do not invent physical pages you did not inspect or obtain from `link`. -- Do not track a total page-count budget. Limits are **token budgets** and - **max_rounds** in the payload. A per-call page cap is only a batch-size limit. - -## Mandatory first step — partition regimes - -Inspect every `page_number` label on the current TOC entries and partition them -into **page-numbering regimes** (distinct numbering systems / label shapes: -decimal digits, roman numerals, prefixed folio labels, etc.). - -- Do not mix samples across regimes when computing an offset. -- Run the same initial-calibration procedure independently for each regime that - has usable entries. - -## Phase 1 — Initial offset (your job via tools) - -For each regime: - -1. Select a small set of entries (prefer spread: early / middle / late when - enough entries exist). -2. Candidate physical page: - - If the entry has `link.physical_page`, use it as the primary candidate. - - Otherwise derive a coarse physical candidate from the printed label and - `page_count`, then confirm with vision. -3. Progressive `inspect.pages` for that title (start small, expand only if needed): - - **1st call**: inspect **1** candidate page only. - - **2nd call** (if miss): inspect up to **3** nearby pages. - - **3rd call** (if still miss): inspect up to **5** nearby pages. - Never open with a full 5-page batch when a single page has not been tried. -4. Compute `offset = physical - printed` using this regime’s interpretation of - the printed label. -5. Submit **candidate** offsets. Do not treat Phase 1 alone as a finished - coarse-structure calibration. - -If `inspect.pages` returns budget exhausted, or rounds run out before a reliable -offset: treat that sample / regime as **not found**, submit whatever regimes you -already confirmed (or `status=failed`), and let production fallback handle the -rest. Do not guess pages. - -## Phase 2 — Completion (deterministic after submit) - -Not your job and not yours to describe. After submit, production completes each -regime independently (prune → tail verify → binary search → small-step -recalibrate), merges the regimes by physical page, and emits the -`SkeletonAnchor`. It recomputes segment coverage, per-regime status and the -no-TOC entry set itself, so do not submit those. - -## Tools - -- `inspect.pages`: primary tool for Phase 1. Open physical pages, render, answer - your question. Prefer the progressive 1→3→5 schedule above. Per-call page - count is capped; overall spend is limited by the calibration visual token - budget and `max_rounds`. -- `calibration.submit`: finish Phase 1. Pass the result under - `tool_args.result` (or result fields directly in `tool_args`). - -## Output rules - -Submit exactly the fields in the `calibration.submit` schema — `status`, -`regimes`, `notes` — and nothing else: - -- Per regime: `kind` and the candidate `offset`. Add `entry_indices` only when - the regime is not simply the entries whose printed-label shape matches `kind`, - and `samples` (`title` + `physical`) only for anchors you actually confirmed. -- `notes`: one short sentence saying why. When you found no offset, submit - `status=failed` and say why in that one sentence. -- Keep `kind` values consistent within one run (`decimal`, `roman`, `prefixed`, - or `other`). -- Anything else — per-regime status, segment coverage, no-TOC entries, tool call - counts, region index — is recomputed after submit; emitting it only risks the - submit being cut off by the output limit, which ends the run with no result. -- Stay within the token / round budgets announced in the payload. diff --git a/apps/worker/app/services/document_agent/agents/calibration/__init__.py b/apps/worker/app/services/document_agent/agents/calibration/__init__.py deleted file mode 100644 index 5c7f30b9f..000000000 --- a/apps/worker/app/services/document_agent/agents/calibration/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Calibration SubAgent package.""" - -from app.services.document_agent.agents.calibration.loop import ( - run_calibration_agent, - run_calibration_for_all_regions, -) -from app.services.document_agent.agents.calibration.procedure import ( - build_calibration_payload, - finalize_calibration_result, -) -from app.services.document_agent.agents.calibration.service import calibrate_offset -from app.services.document_agent.agents.calibration.types import CalibrationResult - -__all__ = [ - "CalibrationResult", - "build_calibration_payload", - "calibrate_offset", - "finalize_calibration_result", - "run_calibration_agent", - "run_calibration_for_all_regions", -] diff --git a/apps/worker/app/services/document_agent/agents/calibration/loop.py b/apps/worker/app/services/document_agent/agents/calibration/loop.py deleted file mode 100644 index d43c12431..000000000 --- a/apps/worker/app/services/document_agent/agents/calibration/loop.py +++ /dev/null @@ -1,541 +0,0 @@ -"""ReAct loop for the calibration SubAgent.""" - -from __future__ import annotations - -import json -import os -import time -from pathlib import Path -from typing import Any - -from loguru import logger - -from app.services.document_agent.agents.calibration.tools import ( - build_calibration_registry, - strip_toc_links, -) -from app.services.document_agent.agents.calibration.procedure import ( - build_calibration_payload, - finalize_calibration_result, -) -from app.services.document_agent.agents.calibration.types import ( - FAILURE_BUDGET_EXHAUSTED, - FAILURE_INVALID_JSON, - FAILURE_LLM_ERROR, - FAILURE_MAX_ROUNDS, - FAILURE_MODEL_MISSING, - FAILURE_NO_OFFSET, - FAILURE_TOC_EMPTY, - CalibrationResult, - calibration_result_from_dict, -) -from app.services.document_agent.budget import BudgetTracker, StageEnvelope -from app.services.document_agent.manifest import ToolContext, ToolResult -from app.services.document_agent.state import AgentBlackboard -from app.services.document_agent.structure.anchoring_primitives import ( - deserialize_skeleton_anchor, - serialize_skeleton_anchor, -) -from shared.utils.token_estimate import estimate_tokens - -_SKILL_PATH = Path(__file__).resolve().parent / "SKILL.md" - -_DECISION_MAX_TOKENS = 2500 - -_DECISION_INSTRUCTIONS = """ -You are the calibration SubAgent. Follow the Skill strictly. -Each turn return a JSON object with keys: - action: "tool_call" - rationale: string - tool_name: one of the available tools - tool_args: object -Your job is Phase 1 only: partition regimes and find candidate offsets via -inspect.pages, then call calibration.submit. -Phase 2 (tail verify, binary search, small-step recalibrate) runs automatically -after submit. Do not use a fixed post-TOC page window. -Hard limits are token budgets and max_rounds — not a total page-count ledger. -Emit exactly the fields in the calibration.submit schema and nothing else; an -oversized submit is cut off by the output limit and ends the run. -Include the word json in your response. -""".strip() - - -def _default_calibration_budget() -> BudgetTracker: - return BudgetTracker( - plan_budget=int(os.environ.get("PARSE_AGENT_PLAN_BUDGET", "50000")), - visual_budget=int(os.environ.get("PARSE_AGENT_VISUAL_BUDGET", "120000")), - visual_stage_envelopes={ - "calibration": StageEnvelope( - min_guarantee=int( - os.environ.get("PARSE_AGENT_CALIBRATION_MIN_BUDGET", "12000") - ), - cap=int(os.environ.get("PARSE_AGENT_CALIBRATION_CAP", "40000")), - ), - }, - ) - - -def _load_skill() -> str: - return _SKILL_PATH.read_text(encoding="utf-8") - - -def _parse_decision(raw: str) -> dict[str, Any]: - data = json.loads(raw) - if not isinstance(data, dict): - return {"tool_name": None, "tool_args": {}, "rationale": "invalid decision"} - tool_name = data.get("tool_name") or data.get("name") or data.get("tool") - tool_args = data.get("tool_args") or data.get("arguments") or data.get("args") or {} - if not isinstance(tool_args, dict): - tool_args = {} - return { - "tool_name": tool_name, - "tool_args": dict(tool_args), - "rationale": str(data.get("rationale") or ""), - } - - -def _attach_history(result: CalibrationResult, history: list[dict[str, Any]]) -> CalibrationResult: - result.history_tail = history[-12:] - return result - - -def _toc_region_payload( - hierarchies: list[dict[str, Any]], - region_index: int, -) -> dict[str, Any]: - if region_index < 0 or region_index >= len(hierarchies): - raise IndexError(f"region_index out of range: {region_index}") - region = hierarchies[region_index] - entries = region.get("toc_with_level") if isinstance(region, dict) else None - return { - "region_index": region_index, - "toc_range": region.get("toc_range") if isinstance(region, dict) else None, - "entries": entries if isinstance(entries, list) else [], - } - - -def run_calibration_phase1( - *, - ctx: ToolContext, - toc_hierarchies: list[dict[str, Any]], - region_index: int = 0, - page_count: int | None = None, - no_links: bool = False, - max_rounds: int = 16, - inspect_page_cap: int = 5, -) -> CalibrationResult: - """Agent Phase-1 only: partition regimes + candidate offsets, then submit. - - Reuses the caller's ``ToolContext`` (budget / pdf / settings). Does **not** - run production Phase-2 bulk anchoring. - - Hard limits: planner/visual token budgets + ``max_rounds``. Per-call - ``inspect_page_cap`` is only a batch-size cap (not a total page ledger). - """ - hierarchies = list(toc_hierarchies or []) - if no_links: - hierarchies = strip_toc_links(hierarchies) - if not hierarchies: - return CalibrationResult( - status="failed", - notes="toc_hierarchies empty", - failure_kind=FAILURE_TOC_EMPTY, - ) - region_payload = _toc_region_payload(hierarchies, region_index) - resolved_page_count = int( - page_count or ctx.blackboard.page_count or 0 - ) - if resolved_page_count: - ctx.blackboard.page_count = resolved_page_count - - ctx.settings.setdefault("inspect_page_cap", inspect_page_cap) - ctx.settings.setdefault("inspect_visual_stage", "calibration") - - blackboard = ctx.blackboard - blackboard.global_signals["calibration_region_index"] = region_index - blackboard.global_signals["calibration_tool_calls"] = 0 - blackboard.global_signals["calibration_done"] = False - blackboard.global_signals.pop("calibration_result", None) - - registry = build_calibration_registry() - skill = _load_skill() - history: list[dict[str, Any]] = [] - - for round_index in range(max_rounds): - available = registry.openai_specs(blackboard) - snap = ctx.budget.snapshot() if ctx.budget is not None else {} - visual_stages = ( - snap.get("visual_stages") if isinstance(snap, dict) else {} - ) or {} - calib_stage = ( - visual_stages.get("calibration") - if isinstance(visual_stages, dict) - else None - ) - payload = { - "skill": skill, - "page_count": resolved_page_count, - "no_links": no_links, - "budgets": { - "max_rounds": max_rounds, - "round_index": round_index, - "rounds_remaining": max_rounds - round_index, - "inspect_page_cap_per_call": int( - ctx.settings.get("inspect_page_cap") or inspect_page_cap - ), - "calibration_visual": calib_stage, - "plan": snap.get("plan") if isinstance(snap, dict) else None, - "visual": snap.get("visual") if isinstance(snap, dict) else None, - }, - "toc_region": region_payload, - "history_tail": history[-8:], - "available_tools": available, - } - prompt = _DECISION_INSTRUCTIONS + "\nPayload:\n" + json.dumps( - payload, ensure_ascii=False - ) - model = ctx.settings.get("model") or ctx.settings.get("vlm_model") - if not model: - return _attach_history( - CalibrationResult( - status="failed", - notes="planner model missing", - failure_kind=FAILURE_MODEL_MISSING, - region_index=region_index, - ), - history, - ) - - est = estimate_tokens(prompt) - if not ctx.budget.try_reserve("plan", est): - return _attach_history( - CalibrationResult( - status="failed", - notes="planner budget exhausted", - failure_kind=FAILURE_BUDGET_EXHAUSTED, - region_index=region_index, - tool_calls=int( - blackboard.global_signals.get("calibration_tool_calls") or 0 - ), - ), - history, - ) - - try: - from shared.services.ai.llm_overrides import get_text_client - - client, model = get_text_client(requested_model=str(model)) - raw, usage = client.chat_completion_with_usage( - messages=[{"role": "user", "content": prompt}], - model=model, - temperature=0.0, - max_tokens=_DECISION_MAX_TOKENS, - response_format={"type": "json_object"}, - usage_task="calibration.react_loop", - ) - except Exception as exc: - ctx.budget.refund("plan", est=est) - logger.warning("[calibration] llm call failed round={}: {}", round_index, exc) - return _attach_history( - CalibrationResult( - status="failed", - notes=f"llm call failed: {exc}", - failure_kind=FAILURE_LLM_ERROR, - region_index=region_index, - tool_calls=int( - blackboard.global_signals.get("calibration_tool_calls") or 0 - ), - ), - history, - ) - - ctx.budget.commit("plan", actual=usage.get("total_tokens", est), est=est) - try: - decision = _parse_decision(raw) - except json.JSONDecodeError as exc: - history.append( - { - "round": round_index, - "error": f"decision output not parseable: {exc}", - "completion_tokens": usage.get("completion_tokens"), - "max_tokens": _DECISION_MAX_TOKENS, - } - ) - logger.warning( - "[calibration] decision output not parseable round={} " - "completion_tokens={} max_tokens={}: {}", - round_index, - usage.get("completion_tokens"), - _DECISION_MAX_TOKENS, - exc, - ) - return _attach_history( - CalibrationResult( - status="failed", - notes=f"decision output not parseable: {exc}", - failure_kind=FAILURE_INVALID_JSON, - region_index=region_index, - tool_calls=int( - blackboard.global_signals.get("calibration_tool_calls") or 0 - ), - ), - history, - ) - - tool_name = str(decision.get("tool_name") or "").strip() - tool_args = dict(decision.get("tool_args") or {}) - if not tool_name: - history.append( - { - "round": round_index, - "error": "missing tool_name", - "decision": decision, - } - ) - continue - - tool_result: ToolResult = registry.dispatch(tool_name, ctx, tool_args) - blackboard.global_signals["calibration_tool_calls"] = ( - int(blackboard.global_signals.get("calibration_tool_calls") or 0) + 1 - ) - history.append( - { - "round": round_index, - "rationale": decision.get("rationale"), - "tool_name": tool_name, - "tool_args": tool_args, - "tool_status": tool_result.status, - "tool_payload": tool_result.output_summary - if tool_result.status == "ok" - else tool_result.payload, - "tool_error": tool_result.error, - } - ) - logger.info( - "[calibration] region={} round={} tool={} status={}", - region_index, - round_index, - tool_name, - tool_result.status, - ) - - if tool_result.status == "error" and "budget exhausted" in str( - tool_result.error or "" - ).lower(): - return _attach_history( - CalibrationResult( - status="failed", - notes=f"budget exhausted: {tool_result.error}", - failure_kind=FAILURE_BUDGET_EXHAUSTED, - region_index=region_index, - tool_calls=int( - blackboard.global_signals.get("calibration_tool_calls") or 0 - ), - ), - history, - ) - - if blackboard.global_signals.get("calibration_done"): - raw_result = blackboard.global_signals.get("calibration_result") or {} - if isinstance(raw_result, dict): - parsed = calibration_result_from_dict(raw_result) - parsed.region_index = region_index - parsed.tool_calls = int( - blackboard.global_signals.get("calibration_tool_calls") or 0 - ) - return _attach_history(parsed, history) - - return _attach_history( - CalibrationResult( - status="failed", - notes="max rounds reached without calibration.submit", - failure_kind=FAILURE_MAX_ROUNDS, - region_index=region_index, - tool_calls=int(blackboard.global_signals.get("calibration_tool_calls") or 0), - ), - history, - ) - - -def run_calibration_agent( - *, - pdf_path: str, - page_count: int, - toc_hierarchies: list[dict[str, Any]], - region_index: int = 0, - output_dir: str, - vlm_model: str | None = None, - planner_model: str | None = None, - no_links: bool = False, - max_rounds: int = 16, - inspect_page_cap: int = 5, - budget: BudgetTracker | None = None, - page_texts: dict[int, str] | None = None, - body_pages: list[int] | None = None, -) -> tuple[CalibrationResult, dict[str, Any]]: - """Debug/full path: Phase-1 agent + production Phase-2 finalize.""" - hierarchies = list(toc_hierarchies or []) - if no_links: - hierarchies = strip_toc_links(hierarchies) - region_hierarchies = [hierarchies[region_index]] if hierarchies else [] - region_payload = _toc_region_payload(hierarchies, region_index) if hierarchies else { - "entries": [] - } - - blackboard = AgentBlackboard() - blackboard.page_count = page_count - if page_texts: - blackboard.page_full_text_cache = dict(page_texts) - - ctx = ToolContext( - pdf_path=pdf_path, - job_id=f"calibration-region-{region_index}", - blackboard=blackboard, - budget=budget or _default_calibration_budget(), - trace=None, - output_dir=output_dir, - settings={ - "vlm_model": vlm_model or "", - "model": planner_model or vlm_model or "", - "inspect_page_cap": inspect_page_cap, - "inspect_visual_stage": "calibration", - }, - ) - - phase1 = run_calibration_phase1( - ctx=ctx, - toc_hierarchies=hierarchies, - region_index=region_index, - page_count=page_count, - no_links=False, # already stripped above when requested - max_rounds=max_rounds, - inspect_page_cap=inspect_page_cap, - ) - if phase1.status == "failed" and not phase1.regimes: - return phase1, {} - - _working, anchor, finalized = finalize_calibration_result( - result=phase1, - entries=list(region_payload.get("entries") or []), - toc_hierarchies=region_hierarchies, - ctx=ctx, - page_count=page_count, - page_texts=page_texts, - body_pages=body_pages, - ) - finalized.history_tail = list(phase1.history_tail) - return finalized, serialize_skeleton_anchor(anchor) - - -def run_calibration_for_all_regions( - *, - pdf_path: str, - page_count: int, - toc_hierarchies: list[dict[str, Any]], - output_dir: str, - vlm_model: str | None = None, - planner_model: str | None = None, - no_links: bool = False, - max_rounds: int = 16, - budget: BudgetTracker | None = None, - page_texts: dict[int, str] | None = None, - body_pages: list[int] | None = None, -) -> dict[str, Any]: - """Calibrate each TOC region; return production SkeletonAnchor-shaped payload.""" - hierarchies = list(toc_hierarchies or []) - if not hierarchies: - return { - "offset": None, - "offset_status": "failed", - "match_overrides": {}, - "null_page_report": [], - "bulk_count": 0, - "pruned_count": 0, - "locate_agent": "offset_only", - "status": "failed", - "regimes": [], - "regions": [], - "notes": "toc_hierarchies empty", - "failure_kind": FAILURE_TOC_EMPTY, - "tool_calls": 0, - "no_links": no_links, - } - - region_results: list[dict[str, Any]] = [] - all_regimes: list[dict[str, Any]] = [] - tool_calls = 0 - primary_anchor: dict[str, Any] | None = None - primary_result: CalibrationResult | None = None - - for idx in range(len(hierarchies)): - t0 = time.time() - result, anchor_dict = run_calibration_agent( - pdf_path=pdf_path, - page_count=page_count, - toc_hierarchies=hierarchies, - region_index=idx, - output_dir=output_dir, - vlm_model=vlm_model, - planner_model=planner_model, - no_links=no_links, - max_rounds=max_rounds, - budget=budget, - page_texts=page_texts, - body_pages=body_pages, - ) - payload = result.to_dict() - payload["elapsed_s"] = round(time.time() - t0, 2) - payload["skeleton_anchor"] = anchor_dict - region_results.append(payload) - tool_calls += int(result.tool_calls or 0) - for regime in payload.get("regimes") or []: - if isinstance(regime, dict): - tagged = dict(regime) - tagged["region_index"] = idx - all_regimes.append(tagged) - if primary_anchor is None and anchor_dict.get("offset") is not None: - primary_anchor = anchor_dict - primary_result = result - - if primary_anchor is None: - primary_anchor = { - "offset": None, - "offset_status": "failed", - "match_overrides": {}, - "null_page_report": [], - "bulk_count": 0, - "pruned_count": 0, - "locate_agent": "offset_only", - } - if primary_result is None: - primary_result = CalibrationResult( - status="failed", - notes="no region produced offset", - failure_kind=FAILURE_NO_OFFSET, - ) - - anchor = deserialize_skeleton_anchor(primary_anchor) - status = ( - "ok" - if anchor.offset is not None and int(anchor.bulk_count or 0) > 0 - else "failed" - ) - merged = build_calibration_payload( - anchor=anchor, - result=CalibrationResult( - status=status, - regimes=[], - offset=anchor.offset, - offset_status=anchor.offset_status, - tool_calls=tool_calls, - notes=primary_result.notes, - failure_kind=primary_result.failure_kind, - ), - no_links=no_links, - region_payloads=region_results, - tool_calls=tool_calls, - ) - merged["status"] = status - merged["regimes"] = all_regimes - merged["regions"] = region_results - return merged diff --git a/apps/worker/app/services/document_agent/agents/calibration/tools.py b/apps/worker/app/services/document_agent/agents/calibration/tools.py deleted file mode 100644 index 952a6e86c..000000000 --- a/apps/worker/app/services/document_agent/agents/calibration/tools.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Tools for the calibration SubAgent (local registry, not PROFILE gates).""" - -from __future__ import annotations - -import time -from typing import Any - -from app.services.document_agent.agents.calibration.types import ( - FAILURE_NO_OFFSET, - calibration_result_from_dict, -) -from app.services.document_agent.manifest import ToolContext, ToolResult -from app.services.document_agent.registry import ToolRegistry, ToolSpec -from app.services.document_agent.tools.inspect_pages import inspect_pages - - -def build_calibration_registry() -> ToolRegistry: - registry = ToolRegistry() - registry.register( - ToolSpec( - name="inspect.pages", - description=( - "Open one or more physical PDF pages, render them, and answer " - "the given question about those pages." - ), - parameters={ - "type": "object", - "properties": { - "pages": { - "type": "array", - "items": {"type": "integer"}, - "description": "1-based physical page numbers", - }, - "question": { - "type": "string", - "description": "Question to answer from the rendered pages", - }, - }, - "required": ["pages", "question"], - }, - preconditions=(), - handler=_calibration_inspect_pages, - ) - ) - registry.register( - ToolSpec( - name="calibration.submit", - description=( - "Submit the Phase-1 result and finish. Only the fields below are " - "read; Phase-2 recomputes everything else." - ), - parameters={ - "type": "object", - "properties": { - "result": { - "type": "object", - "properties": { - "status": { - "type": "string", - "enum": ["ok", "failed"], - }, - "regimes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "description": ( - "Page-numbering system of this " - "regime's printed labels" - ), - }, - "offset": { - "type": "integer", - "description": "physical - printed", - }, - "entry_indices": { - "type": "array", - "items": {"type": "integer"}, - "description": ( - "0-based indices into " - "toc_region.entries; omit when the " - "regime is exactly the entries " - "whose label shape matches kind" - ), - }, - "samples": { - "type": "array", - "items": { - "type": "object", - "properties": { - "title": {"type": "string"}, - "physical": {"type": "integer"}, - }, - "required": ["title", "physical"], - }, - "description": ( - "Anchors you confirmed with " - "inspect.pages, so Phase-2 does not " - "re-verify them" - ), - }, - }, - "required": ["kind", "offset"], - }, - }, - "notes": { - "type": "string", - "description": "One short sentence: why this result", - }, - }, - "required": ["status", "regimes"], - }, - }, - "required": ["result"], - }, - preconditions=(), - handler=calibration_submit, - ) - ) - return registry - - -def _calibration_inspect_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: - merged = dict(args) - merged.setdefault("folder_name", "calibration_inspect") - merged.setdefault("prefix", "calib") - merged.setdefault("usage_task", "calibration.inspect_pages") - merged.setdefault("visual_stage", "calibration") - return inspect_pages(ctx, merged) - - -def calibration_submit(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: - start = time.monotonic() - raw = args.get("result") - if not isinstance(raw, dict): - if any(key in args for key in ("status", "regimes")): - raw = { - key: args.get(key) - for key in ("status", "regimes", "notes") - if key in args - } - else: - return ToolResult( - status="error", - error="calibration.submit requires result object", - latency_ms=int((time.monotonic() - start) * 1000), - ) - result = calibration_result_from_dict(raw) - if not result.regimes and result.status == "ok": - return ToolResult( - status="error", - error="ok result must include regimes", - latency_ms=int((time.monotonic() - start) * 1000), - ) - if not result.regimes: - result.failure_kind = FAILURE_NO_OFFSET - tool_calls = int(ctx.blackboard.global_signals.get("calibration_tool_calls") or 0) - result.tool_calls = tool_calls - region_index = ctx.blackboard.global_signals.get("calibration_region_index") - if region_index is not None: - result.region_index = int(region_index) - ctx.blackboard.global_signals["calibration_result"] = result.to_dict() - ctx.blackboard.global_signals["calibration_done"] = True - return ToolResult( - status="ok", - payload=result.to_dict(), - latency_ms=int((time.monotonic() - start) * 1000), - output_summary={"status": result.status, "regimes": len(result.regimes)}, - ) - - -def strip_toc_links(hierarchies: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Return hierarchies with entry ``link`` keys removed.""" - out: list[dict[str, Any]] = [] - for hierarchy in hierarchies: - if not isinstance(hierarchy, dict): - continue - cloned = dict(hierarchy) - entries = hierarchy.get("toc_with_level") - if isinstance(entries, list): - new_entries: list[dict[str, Any]] = [] - for entry in entries: - if not isinstance(entry, dict): - continue - item = dict(entry) - item.pop("link", None) - new_entries.append(item) - cloned["toc_with_level"] = new_entries - out.append(cloned) - return out diff --git a/apps/worker/app/services/document_agent/agents/calibration/types.py b/apps/worker/app/services/document_agent/agents/calibration/types.py deleted file mode 100644 index 5970efd99..000000000 --- a/apps/worker/app/services/document_agent/agents/calibration/types.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Calibration SubAgent result types. - -The ``calibration.submit`` payload carries only what Phase-2 cannot recompute: -``status``, per-regime numbering ``kind`` + candidate ``offset`` (plus the -anchor ``samples`` already confirmed by vision), and one short ``notes`` reason. -``segments`` / ``no_toc_entry_indices`` / ``offset_status`` / per-regime -``notes`` / ``tool_calls`` / ``region_index`` are Phase-2 or harness outputs and -are never read back from a submit payload. -""" - -from __future__ import annotations - -from dataclasses import asdict, dataclass, field -from typing import Any - -# Failure classes recorded on ``CalibrationResult.failure_kind``, one per -# failure exit of the ReAct loop. ``INVALID_JSON`` means the decision payload -# did not parse (history carries completion_tokens vs max_tokens so truncation -# can be diagnosed offline); the rest mean the episode ran without an offset. -FAILURE_INVALID_JSON = "invalid_json" -FAILURE_LLM_ERROR = "llm_error" -FAILURE_MODEL_MISSING = "model_missing" -FAILURE_BUDGET_EXHAUSTED = "budget_exhausted" -FAILURE_MAX_ROUNDS = "max_rounds" -FAILURE_NO_OFFSET = "no_offset" -FAILURE_TOC_EMPTY = "toc_empty" - - -@dataclass -class CalibrationSample: - """A printed→physical anchor the agent confirmed with ``inspect.pages``.""" - - title: str - physical: int | None = None - - -@dataclass -class CalibrationSegment: - """A contiguous leaf range that fully completed Phase-2 for one offset.""" - - offset: int - leaf_start: int - leaf_end: int - entry_indices: list[int] = field(default_factory=list) - status: str = "ok" - - -@dataclass -class CalibrationRegime: - kind: str - offset: int | None = None - entry_indices: list[int] = field(default_factory=list) - samples: list[CalibrationSample] = field(default_factory=list) - # Phase-2 outputs below; never parsed from the agent submit payload. - offset_status: str = "failed" - segments: list[CalibrationSegment] = field(default_factory=list) - no_toc_entry_indices: list[int] = field(default_factory=list) - notes: str = "" - - -@dataclass -class CalibrationResult: - status: str - regimes: list[CalibrationRegime] = field(default_factory=list) - offset: int | None = None - offset_status: str = "failed" - tool_calls: int = 0 - notes: str = "" - # Empty on success; otherwise one of the FAILURE_* constants, so a submit - # that never parsed is never read as "this document has no offset". - failure_kind: str = "" - region_index: int | None = None - # Debug-only trail from the ReAct loop (not part of submit schema). - history_tail: list[dict[str, Any]] = field(default_factory=list) - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -def _as_optional_int(value: Any) -> int | None: - if value is None or isinstance(value, bool): - return None - if isinstance(value, int): - return value - if isinstance(value, float): - return int(value) - if isinstance(value, str): - text = value.strip() - if not text: - return None - try: - return int(text) - except ValueError: - return None - return None - - -def _as_int_list(value: Any) -> list[int]: - if not isinstance(value, list): - return [] - out: list[int] = [] - for item in value: - parsed = _as_optional_int(item) - if parsed is not None: - out.append(parsed) - return out - - -def calibration_result_from_dict(data: dict[str, Any]) -> CalibrationResult: - """Parse the minimal submit payload; ignore anything Phase-2 recomputes.""" - regimes: list[CalibrationRegime] = [] - for raw in data.get("regimes") or []: - if not isinstance(raw, dict): - continue - samples = [ - CalibrationSample( - title=str(s.get("title") or ""), - physical=_as_optional_int(s.get("physical")), - ) - for s in (raw.get("samples") or []) - if isinstance(s, dict) - ] - regimes.append( - CalibrationRegime( - kind=str(raw.get("kind") or "other"), - offset=_as_optional_int(raw.get("offset")), - entry_indices=_as_int_list(raw.get("entry_indices")), - samples=samples, - ) - ) - return CalibrationResult( - status=str(data.get("status") or "failed"), - regimes=regimes, - notes=str(data.get("notes") or ""), - failure_kind=str(data.get("failure_kind") or ""), - ) diff --git a/apps/worker/app/services/document_agent/bootstrap/__init__.py b/apps/worker/app/services/document_agent/bootstrap/__init__.py index 2a490a4bd..c58a6e93f 100644 --- a/apps/worker/app/services/document_agent/bootstrap/__init__.py +++ b/apps/worker/app/services/document_agent/bootstrap/__init__.py @@ -1,4 +1,4 @@ -"""Deterministic bootstrap steps for the document profile agent.""" +"""Deterministic bootstrap steps for the document profile workflow.""" from app.services.document_agent.bootstrap.aggregate_stats import aggregate_doc_stats from app.services.document_agent.bootstrap.classify import classify_page_kinds diff --git a/apps/worker/app/services/document_agent/bootstrap/aggregate_stats.py b/apps/worker/app/services/document_agent/bootstrap/aggregate_stats.py index 96a119421..f986bc095 100644 --- a/apps/worker/app/services/document_agent/bootstrap/aggregate_stats.py +++ b/apps/worker/app/services/document_agent/bootstrap/aggregate_stats.py @@ -119,7 +119,6 @@ def aggregate_doc_stats(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult: ctx.blackboard.extrema_pages = deduped_extrema ctx.blackboard.global_signals["doc_stats"] = stats ctx.blackboard.global_signals["doc_shape"] = doc_shape - ctx.blackboard.global_signals["extrema_pages"] = deduped_extrema ctx.blackboard.global_signals["extrema_samples"] = extrema_samples return ToolResult( status="ok", diff --git a/apps/worker/app/services/document_agent/budget.py b/apps/worker/app/services/document_agent/budget.py deleted file mode 100644 index 3ecdf867f..000000000 --- a/apps/worker/app/services/document_agent/budget.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Small synchronous budget tracker for parse-side agent planning.""" - -from __future__ import annotations - -import threading -from dataclasses import dataclass -from typing import Literal - - -BudgetStage = Literal[ - "toc_confirm", - "coarse_planner", - "structural_react", - "calibration", - "page_tagging", -] - - -@dataclass -class BudgetPool: - capacity: int - used: int = 0 - reserved: int = 0 - - @property - def remaining(self) -> int: - return max(self.capacity - self.used - self.reserved, 0) - - -@dataclass(frozen=True) -class StageEnvelope: - min_guarantee: int = 0 - cap: int | None = None - - -@dataclass -class StageUsage: - used: int = 0 - reserved: int = 0 - - @property - def committed(self) -> int: - return self.used + self.reserved - - -class BudgetTracker: - """A minimal synchronous ledger with plan and visual pools.""" - - def __init__( - self, - *, - plan_budget: int = 5000, - visual_budget: int = 8000, - visual_stage_envelopes: dict[str, StageEnvelope] | None = None, - ) -> None: - self._lock = threading.Lock() - self._plan = BudgetPool(capacity=max(int(plan_budget), 0)) - self._visual = BudgetPool(capacity=max(int(visual_budget), 0)) - self._visual_stage_envelopes = visual_stage_envelopes or {} - self._visual_stage_usage: dict[str, StageUsage] = { - stage: StageUsage() for stage in self._visual_stage_envelopes - } - - def try_reserve(self, pool: str, est: int, *, stage: str | None = None) -> bool: - if pool not in {"plan", "visual"}: - return True - est = max(int(est), 0) - with self._lock: - budget_pool = self._pool(pool) - if budget_pool.remaining < est: - return False - if pool == "visual" and stage and not self._can_reserve_visual_stage(stage, est): - return False - budget_pool.reserved += est - if pool == "visual" and stage: - self._stage_usage(stage).reserved += est - return True - - def commit( - self, - pool: str, - *, - actual: int, - est: int, - stage: str | None = None, - ) -> None: - if pool not in {"plan", "visual"}: - return - est = max(int(est), 0) - actual = max(int(actual), 0) - with self._lock: - budget_pool = self._pool(pool) - budget_pool.reserved = max(budget_pool.reserved - est, 0) - budget_pool.used = min(budget_pool.capacity, budget_pool.used + actual) - if pool == "visual" and stage: - stage_usage = self._stage_usage(stage) - stage_usage.reserved = max(stage_usage.reserved - est, 0) - stage_usage.used += actual - - def refund(self, pool: str, *, est: int, stage: str | None = None) -> None: - if pool not in {"plan", "visual"}: - return - est = max(int(est), 0) - with self._lock: - budget_pool = self._pool(pool) - budget_pool.reserved = max(budget_pool.reserved - est, 0) - if pool == "visual" and stage: - stage_usage = self._stage_usage(stage) - stage_usage.reserved = max(stage_usage.reserved - est, 0) - - def _pool(self, pool: str) -> BudgetPool: - return self._visual if pool == "visual" else self._plan - - def _stage_usage(self, stage: str) -> StageUsage: - if stage not in self._visual_stage_usage: - self._visual_stage_usage[stage] = StageUsage() - return self._visual_stage_usage[stage] - - def _can_reserve_visual_stage(self, stage: str, est: int) -> bool: - envelope = self._visual_stage_envelopes.get(stage) - if envelope is None: - return True - - stage_usage = self._stage_usage(stage) - stage_committed_after_reserve = stage_usage.committed + est - if envelope.cap is not None and stage_committed_after_reserve > envelope.cap: - return False - - reserved_by_other_stages = 0 - for other_stage, other_envelope in self._visual_stage_envelopes.items(): - if other_stage == stage: - continue - other_usage = self._stage_usage(other_stage) - if other_usage.committed >= other_envelope.min_guarantee: - continue - reserved_by_other_stages += other_envelope.min_guarantee - other_usage.committed - - return self._visual.remaining - est >= reserved_by_other_stages - - def _pool_snapshot(self, pool: BudgetPool) -> dict[str, int]: - return { - "capacity": pool.capacity, - "used": pool.used, - "reserved": pool.reserved, - "remaining": pool.remaining, - } - - def snapshot(self) -> dict[str, object]: - with self._lock: - return { - "plan": self._pool_snapshot(self._plan), - "visual": self._pool_snapshot(self._visual), - "visual_stages": { - stage: { - "used": usage.used, - "reserved": usage.reserved, - "min_guarantee": self._visual_stage_envelopes.get( - stage, StageEnvelope() - ).min_guarantee, - "cap": self._visual_stage_envelopes.get( - stage, StageEnvelope() - ).cap, - } - for stage, usage in sorted(self._visual_stage_usage.items()) - }, - } - - def fork(self, ratio: float) -> "BudgetTracker": - """Create an independent child tracker with proportional remaining budget.""" - ratio = max(0.0, min(1.0, float(ratio))) - with self._lock: - child_envelopes: dict[str, StageEnvelope] = {} - for stage, envelope in self._visual_stage_envelopes.items(): - usage = self._visual_stage_usage.get(stage, StageUsage()) - remaining_guarantee = max(envelope.min_guarantee - usage.committed, 0) - child_envelopes[stage] = StageEnvelope( - min_guarantee=int(remaining_guarantee * ratio), - cap=int(envelope.cap * ratio) if envelope.cap is not None else None, - ) - - return BudgetTracker( - plan_budget=int(self._plan.remaining * ratio), - visual_budget=int(self._visual.remaining * ratio), - visual_stage_envelopes=child_envelopes, - ) diff --git a/apps/worker/app/services/document_agent/calibration/__init__.py b/apps/worker/app/services/document_agent/calibration/__init__.py new file mode 100644 index 000000000..01d41dc1c --- /dev/null +++ b/apps/worker/app/services/document_agent/calibration/__init__.py @@ -0,0 +1,64 @@ +"""Calibration package: deterministic Phase-1 scan + production Phase-2. + +Keep this module import-light: ``structure.section_page_verify`` imports +``calibration.prompts``, and heavy eager imports here recreate a circular +import through ``tools`` → ``anchoring_primitives``. + +Public names are resolved lazily via ``__getattr__`` (no ``__all__`` list of +undefined symbols — that trips CodeQL / pyright). +""" + +from typing import Any + +_LAZY_EXPORTS = ( + "CalibrationResult", + "build_calibration_payload", + "calibrate_offset", + "finalize_calibration_result", + "run_calibration_phase1", + "scan_title_forward", +) + + +def __getattr__(name: str) -> Any: + if name == "CalibrationResult": + from app.services.document_agent.calibration.types import ( + CalibrationResult, + ) + + return CalibrationResult + if name == "build_calibration_payload": + from app.services.document_agent.calibration.procedure import ( + build_calibration_payload, + ) + + return build_calibration_payload + if name == "calibrate_offset": + from app.services.document_agent.calibration.service import ( + calibrate_offset, + ) + + return calibrate_offset + if name == "finalize_calibration_result": + from app.services.document_agent.calibration.procedure import ( + finalize_calibration_result, + ) + + return finalize_calibration_result + if name == "run_calibration_phase1": + from app.services.document_agent.calibration.phase1 import ( + run_calibration_phase1, + ) + + return run_calibration_phase1 + if name == "scan_title_forward": + from app.services.document_agent.calibration.scan import ( + scan_title_forward, + ) + + return scan_title_forward + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted({*globals(), *_LAZY_EXPORTS}) diff --git a/apps/worker/app/services/document_agent/agents/calibration/orchestrator.py b/apps/worker/app/services/document_agent/calibration/orchestrator.py similarity index 91% rename from apps/worker/app/services/document_agent/agents/calibration/orchestrator.py rename to apps/worker/app/services/document_agent/calibration/orchestrator.py index bb9b80c4c..663afc3c7 100644 --- a/apps/worker/app/services/document_agent/agents/calibration/orchestrator.py +++ b/apps/worker/app/services/document_agent/calibration/orchestrator.py @@ -4,11 +4,11 @@ from typing import Any -from app.services.document_agent.agents.calibration.procedure import ( +from app.services.document_agent.calibration.procedure import ( finalize_calibration_result, flat_toc_entries, ) -from app.services.document_agent.agents.calibration import service +from app.services.document_agent.calibration import service from app.services.document_agent.manifest import ToolContext from app.services.document_agent.structure.hierarchy_locator import TitleNode from app.services.document_agent.structure.anchoring_primitives import ( @@ -28,7 +28,6 @@ def anchor_hierarchy( ) -> tuple[list[TitleNode], SkeletonAnchor]: """Run calibration Phase-1 and the production Phase-2 completion.""" phase1 = service.calibrate_offset( - nodes=nodes, toc_hierarchies=toc_hierarchies, ctx=ctx, page_texts=page_texts, diff --git a/apps/worker/app/services/document_agent/calibration/phase1.py b/apps/worker/app/services/document_agent/calibration/phase1.py new file mode 100644 index 000000000..8d8eba256 --- /dev/null +++ b/apps/worker/app/services/document_agent/calibration/phase1.py @@ -0,0 +1,180 @@ +"""Deterministic calibration Phase-1: regime partition + forward-scan offsets. + +Entries are partitioned by printed-label kind (the same classifier Phase-2 +uses). Each regime takes its first few entries as probes and scans forward from +the page after this TOC region's ``toc_range`` end until one is confirmed; that +single confirmation fixes the regime's candidate offset. Phase-2 owns tail +verification and bulk anchoring. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from loguru import logger + +from app.services.document_agent.calibration.scan import ( + TitleScanResult, + scan_title_forward, +) +from app.services.document_agent.calibration.types import ( + FAILURE_NO_OFFSET, + FAILURE_PAGE_COUNT_MISSING, + FAILURE_TOC_EMPTY, + CalibrationRegime, + CalibrationResult, + CalibrationSample, +) +from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.structure.anchoring_primitives import toc_range_end +from app.services.document_agent.structure.hierarchy_locator import ( + classify_page_number_kind, + parse_printed_page, +) + +PROBES_PER_REGIME = 2 + + +@dataclass(frozen=True) +class _Probe: + title: str + printed: int + + +def _region_entries( + hierarchies: list[dict[str, Any]], + region_index: int, +) -> list[dict[str, Any]]: + if region_index < 0 or region_index >= len(hierarchies): + raise IndexError(f"region_index out of range: {region_index}") + region = hierarchies[region_index] + entries = region.get("toc_with_level") if isinstance(region, dict) else None + return [entry for entry in (entries or []) if isinstance(entry, dict)] + + +def _regime_probes( + entries: list[dict[str, Any]], + *, + limit: int, +) -> dict[str, list[_Probe]]: + """Group entries by printed-label kind, keeping the first ``limit`` per kind.""" + from app.services.document_parser.structure.body_boundary import ( + normalize_heading_text, + ) + + probes: dict[str, list[_Probe]] = {} + for entry in entries: + label = entry.get("page_number") + kind = classify_page_number_kind(label) + if len(probes.get(kind, ())) >= limit: + continue + printed = parse_printed_page(label, kind=kind) + if printed is None: + continue + title = normalize_heading_text(str(entry.get("heading") or "")) + if not title: + continue + probes.setdefault(kind, []).append(_Probe(title=title, printed=printed)) + return probes + + +def run_calibration_phase1( + *, + ctx: ToolContext, + toc_hierarchies: list[dict[str, Any]], + region_index: int = 0, + page_count: int | None = None, + probes_per_regime: int = PROBES_PER_REGIME, +) -> CalibrationResult: + """Find one candidate offset per page-numbering regime in this TOC region.""" + hierarchies = list(toc_hierarchies or []) + if not hierarchies: + return CalibrationResult( + status="failed", + notes="toc_hierarchies empty", + failure_kind=FAILURE_TOC_EMPTY, + region_index=region_index, + ) + + resolved_page_count = int(page_count or ctx.blackboard.page_count or 0) + if not resolved_page_count: + return CalibrationResult( + status="failed", + notes="page_count unknown", + failure_kind=FAILURE_PAGE_COUNT_MISSING, + region_index=region_index, + ) + ctx.blackboard.page_count = resolved_page_count + + region = hierarchies[region_index] + toc_end = toc_range_end(region) if isinstance(region, dict) else None + if toc_end is None: + return CalibrationResult( + status="failed", + notes="toc_range end unknown", + failure_kind=FAILURE_TOC_EMPTY, + region_index=region_index, + ) + scan_start = toc_end + 1 + if scan_start > resolved_page_count: + return CalibrationResult( + status="failed", + notes=f"scan start {scan_start} beyond page_count {resolved_page_count}", + failure_kind=FAILURE_NO_OFFSET, + region_index=region_index, + ) + + probes = _regime_probes( + _region_entries(hierarchies, region_index), limit=probes_per_regime + ) + regimes: list[CalibrationRegime] = [] + scans: list[TitleScanResult] = [] + + for kind, bucket in probes.items(): + for probe in bucket: + scan = scan_title_forward( + ctx=ctx, + title=probe.title, + start_page=scan_start, + page_count=resolved_page_count, + ) + scans.append(scan) + if not scan.found or scan.found_page is None: + continue + regimes.append( + CalibrationRegime( + kind=kind, + offset=scan.found_page - probe.printed, + samples=[ + CalibrationSample(title=probe.title, physical=scan.found_page) + ], + ) + ) + break + + inspect_calls = sum(len(scan.rounds) for scan in scans) + logger.info( + "[calibration.phase1] region={} regimes={} offsets={} inspect_calls={}", + region_index, + len(probes), + [regime.offset for regime in regimes], + inspect_calls, + ) + if not regimes: + return CalibrationResult( + status="failed", + notes=f"no regime confirmed after {inspect_calls} inspect call(s)", + failure_kind=FAILURE_NO_OFFSET, + region_index=region_index, + tool_calls=inspect_calls, + scans=[scan.to_dict() for scan in scans], + ) + return CalibrationResult( + status="ok", + regimes=regimes, + notes=f"{len(regimes)}/{len(probes)} regime(s) confirmed by forward scan", + region_index=region_index, + tool_calls=inspect_calls, + scans=[scan.to_dict() for scan in scans], + ) diff --git a/apps/worker/app/services/document_agent/agents/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py similarity index 94% rename from apps/worker/app/services/document_agent/agents/calibration/procedure.py rename to apps/worker/app/services/document_agent/calibration/procedure.py index 2fac004ea..90dbc78ac 100644 --- a/apps/worker/app/services/document_agent/agents/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -1,6 +1,6 @@ """Phase-2 completion aligned with production anchoring. -After the agent submits candidate regime offsets, this module: +After Phase-1 returns candidate regime offsets, this module: 1. Builds TitleNodes the same way production does 2. Runs Phase-2 **per regime** (prune → bulk/bisect → recalibrate) 3. Merges physical-page ``match_overrides`` across regimes @@ -16,7 +16,7 @@ from loguru import logger -from app.services.document_agent.agents.calibration.types import ( +from app.services.document_agent.calibration.types import ( CalibrationRegime, CalibrationResult, CalibrationSegment, @@ -33,14 +33,12 @@ ) from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, + backfill_parent_offset_matches, locate_null_page_parent_overrides, prune_unanchored_toc_leaves, serialize_skeleton_anchor, ) from app.services.document_agent.structure import anchoring_primitives as _anchoring -from app.services.document_agent.structure.page_locate_agent import ( - verify_section_page_choice, -) # Re-export under prior names so existing imports keep working. normalize_kind = normalize_page_kind @@ -54,20 +52,14 @@ def offset_guided_anchoring( page_count: int, calibration_overrides: dict[tuple[str, ...], TitleMatch], ) -> dict[tuple[str, ...], TitleMatch] | None: - """Forward phase-2 anchoring while preserving the historical patch seam.""" - original = _anchoring.verify_section_page_choice - _anchoring.verify_section_page_choice = verify_section_page_choice - try: - return _anchoring.offset_guided_anchoring( - nodes=nodes, - offset=offset, - ctx=ctx, - page_count=page_count, - calibration_overrides=calibration_overrides, - ) - finally: - _anchoring.verify_section_page_choice = original - + """Forward to production Phase-2 anchoring.""" + return _anchoring.offset_guided_anchoring( + nodes=nodes, + offset=offset, + ctx=ctx, + page_count=page_count, + calibration_overrides=calibration_overrides, + ) def pick_primary_offset(result: CalibrationResult) -> int | None: """Prefer decimal-regime candidate offset; else first regime with an offset.""" @@ -106,14 +98,12 @@ def seed_overrides_from_samples( continue overrides[path] = TitleMatch( page=int(sample.physical), - confidence=0.85, - source="agent_vlm", + source="inspect_vlm", matched_line="", - score=0.85, candidates=[int(sample.physical)], evidence={ "calibration": True, - "method": "agent_phase1", + "method": "phase1_forward_scan", "regime_kind": regime.kind, }, ) @@ -376,6 +366,19 @@ def anchor_hierarchy_from_regimes( if path in surviving_paths } + parent_matches = backfill_parent_offset_matches( + nodes=working, + matches=merged, + page_count=page_count, + ) + if parent_matches: + merged.update(parent_matches) + logger.info( + "[calibration.phase2] parent backfill: {} printed-page TOC parents " + "anchored from descendant offset", + len(parent_matches), + ) + match_overrides, null_page_report = locate_null_page_parent_overrides( nodes=working, match_overrides=merged, @@ -393,7 +396,7 @@ def anchor_hierarchy_from_regimes( else: offset_status = "ok" - locate_agent = ( + locate_method = ( "offset_guided_bulk" if match_overrides and (regime_bulk > 0 or seed) else "offset_only" @@ -407,7 +410,7 @@ def anchor_hierarchy_from_regimes( null_page_report=null_page_report, bulk_count=bulk_count, pruned_count=total_pruned, - locate_agent=locate_agent, + locate_method=locate_method, ) @@ -486,7 +489,7 @@ def _annotate_regimes_from_anchor( no_toc_entry_indices=no_toc, notes=( f"production_bulk={anchor.bulk_count}; " - f"locate_agent={anchor.locate_agent}; " + f"locate_method={anchor.locate_method}; " f"regime_anchored={len(ok_indices)}" ), ) @@ -542,7 +545,7 @@ def finalize_calibration_result( complete = sum(len(r.segments) for r in regimes) notes_parts = [result.notes] if result.notes else [] notes_parts.append( - f"phase2 production locate_agent={anchor.locate_agent} " + f"phase2 production locate_method={anchor.locate_method} " f"bulk={anchor.bulk_count} complete_regime_segments={complete}" ) finalized = CalibrationResult( @@ -554,7 +557,7 @@ def finalize_calibration_result( notes="; ".join(p for p in notes_parts if p), failure_kind=result.failure_kind, region_index=result.region_index, - history_tail=list(result.history_tail), + scans=list(result.scans), ) return working, anchor, finalized @@ -563,7 +566,6 @@ def build_calibration_payload( *, anchor: SkeletonAnchor, result: CalibrationResult, - no_links: bool, region_payloads: list[dict[str, Any]] | None = None, tool_calls: int | None = None, ) -> dict[str, Any]: @@ -580,7 +582,6 @@ def build_calibration_payload( "tool_calls": int(tool_calls if tool_calls is not None else result.tool_calls), "notes": result.notes, "failure_kind": result.failure_kind, - "no_links": no_links, } ) return payload diff --git a/apps/worker/app/services/document_agent/calibration/prompts.py b/apps/worker/app/services/document_agent/calibration/prompts.py new file mode 100644 index 000000000..123910880 --- /dev/null +++ b/apps/worker/app/services/document_agent/calibration/prompts.py @@ -0,0 +1,38 @@ +"""Shared VLM prompts for printed→physical section-start verification.""" + +from __future__ import annotations + +from typing import Any + +SECTION_START_ANSWER_KEYS = { + "found": "boolean, true only when the section heading starts on one of these pages", + "found_page": "number|null, the physical page number where it starts", +} + + +def build_section_start_question(title: str) -> str: + """Ask whether ``title`` starts as a body heading on the provided pages.""" + return ( + f"Does the section titled {title!r} START on one of these pages, as a " + "body heading? A table-of-contents line, a running header or footer, or " + "a passing mention in body text does not count. Report the physical page " + "number printed in the page label above each image." + ) + + +def coerce_found(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return value != 0 + if isinstance(value, str): + return value.strip().lower() in {"true", "1", "yes"} + return False + + +def coerce_found_page(value: Any, *, pages: list[int]) -> int | None: + try: + page = int(value) + except (TypeError, ValueError): + return None + return page if page in pages else None diff --git a/apps/worker/app/services/document_agent/calibration/scan.py b/apps/worker/app/services/document_agent/calibration/scan.py new file mode 100644 index 000000000..95b468e75 --- /dev/null +++ b/apps/worker/app/services/document_agent/calibration/scan.py @@ -0,0 +1,147 @@ +"""Deterministic forward scan for a TOC title via ``inspect.pages``. + +A TOC entry gives a printed page label; the physical page is that candidate or +some page after it. The scan walks forward from the candidate with a widening +window, feeding each round's cursor into the next one, so a miss never re-opens +pages that were already inspected. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from loguru import logger + +from app.services.document_agent.calibration.prompts import ( + SECTION_START_ANSWER_KEYS, + build_section_start_question, + coerce_found, + coerce_found_page, +) +from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.tools.inspect_pages import inspect_pages + +DEFAULT_WINDOW_SCHEDULE: tuple[int, ...] = (2, 4, 6, 10) + + +@dataclass +class ScanRound: + pages: list[int] + found: bool + found_page: int | None = None + error: str = "" + + +@dataclass +class TitleScanResult: + """Typed outcome of one title scan; ``next_start`` is the live cursor.""" + + title: str + found: bool + found_page: int | None + scanned_pages: list[int] + next_start: int | None + rounds: list[ScanRound] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "title": self.title, + "found": self.found, + "found_page": self.found_page, + "scanned_pages": list(self.scanned_pages), + "next_start": self.next_start, + "rounds": [ + { + "pages": list(item.pages), + "found": item.found, + "found_page": item.found_page, + "error": item.error, + } + for item in self.rounds + ], + } + + +def scan_title_forward( + *, + ctx: ToolContext, + title: str, + start_page: int, + page_count: int, + window_schedule: tuple[int, ...] = DEFAULT_WINDOW_SCHEDULE, +) -> TitleScanResult: + """Scan forward from ``start_page`` until the title is found or rounds run out. + + Each round opens ``window_schedule[i]`` consecutive pages starting at the + cursor left by the previous round, so no page is inspected twice. + """ + cursor = max(int(start_page), 1) + scanned: list[int] = [] + rounds: list[ScanRound] = [] + + for size in window_schedule: + if cursor > page_count: + break + pages = [page for page in range(cursor, cursor + size) if page <= page_count] + if not pages: + break + + result = inspect_pages( + ctx, + { + "pages": pages, + "page_cap": len(pages), + "question": build_section_start_question(title), + "answer_keys": SECTION_START_ANSWER_KEYS, + "folder_name": "calibration_scan", + "prefix": "scan", + "usage_task": "calibration.scan_title_forward", + }, + ) + cursor = pages[-1] + 1 + scanned.extend(pages) + + if result.status != "ok": + rounds.append(ScanRound(pages=pages, found=False, error=result.error or "")) + logger.warning( + "[calibration.scan] title={!r} pages={} inspect failed: {}", + title, + pages, + result.error, + ) + break + + fields = (result.payload or {}).get("fields") or {} + found_page = coerce_found_page(fields.get("found_page"), pages=pages) + found = coerce_found(fields.get("found")) and found_page is not None + rounds.append(ScanRound(pages=pages, found=found, found_page=found_page)) + if found: + logger.info( + "[calibration.scan] title={!r} found on page={} after {} round(s)", + title, + found_page, + len(rounds), + ) + return TitleScanResult( + title=title, + found=True, + found_page=found_page, + scanned_pages=scanned, + next_start=cursor if cursor <= page_count else None, + rounds=rounds, + ) + + logger.info( + "[calibration.scan] title={!r} not found in pages={}", + title, + scanned, + ) + return TitleScanResult( + title=title, + found=False, + found_page=None, + scanned_pages=scanned, + next_start=cursor if cursor <= page_count else None, + rounds=rounds, + ) diff --git a/apps/worker/app/services/document_agent/agents/calibration/service.py b/apps/worker/app/services/document_agent/calibration/service.py similarity index 67% rename from apps/worker/app/services/document_agent/agents/calibration/service.py rename to apps/worker/app/services/document_agent/calibration/service.py index 288531000..39a609d77 100644 --- a/apps/worker/app/services/document_agent/agents/calibration/service.py +++ b/apps/worker/app/services/document_agent/calibration/service.py @@ -1,4 +1,4 @@ -"""Production calibration entry: Agent Phase-1 offset discovery. +"""Production calibration entry: Phase-1 offset discovery. Returns a full ``CalibrationResult`` (all regimes). Callers run multi-regime Phase-2 via ``finalize_calibration_result`` / ``anchor_hierarchy``. @@ -10,32 +10,36 @@ from loguru import logger -from app.services.document_agent.agents.calibration.loop import run_calibration_phase1 -from app.services.document_agent.agents.calibration.types import CalibrationResult +from app.services.document_agent.calibration.phase1 import run_calibration_phase1 +from app.services.document_agent.calibration.types import ( + FAILURE_TOC_EMPTY, + CalibrationResult, +) from app.services.document_agent.manifest import ToolContext -from app.services.document_agent.structure.hierarchy_locator import TitleNode def calibrate_offset( *, - nodes: list[TitleNode], toc_hierarchies: list[dict[str, Any]] | None, ctx: ToolContext | None, page_texts: dict[int, str], page_count: int, ) -> CalibrationResult: - """Discover printed→physical offsets via the calibration SubAgent (Phase 1). + """Discover printed→physical offsets by deterministic forward scan (Phase 1). - Returns the full Phase-1 ``CalibrationResult`` including every regime the - agent submitted. Phase-2 (per-regime bulk / bisect / null-page merge) is - owned by ``finalize_calibration_result`` / ``anchor_hierarchy``. + Returns the full Phase-1 ``CalibrationResult`` including every regime that + was confirmed. Phase-2 (per-regime bulk / bisect / null-page merge) is owned + by ``finalize_calibration_result`` / ``anchor_hierarchy``. """ - del nodes # Phase-1 works from toc_hierarchies entries; nodes used in Phase-2. if ctx is None: return CalibrationResult(status="failed", notes="ctx missing") hierarchies = list(toc_hierarchies or []) if not hierarchies: - return CalibrationResult(status="failed", notes="toc_hierarchies empty") + return CalibrationResult( + status="failed", + notes="toc_hierarchies empty", + failure_kind=FAILURE_TOC_EMPTY, + ) if page_count and not ctx.blackboard.page_count: ctx.blackboard.page_count = int(page_count) diff --git a/apps/worker/app/services/document_agent/calibration/types.py b/apps/worker/app/services/document_agent/calibration/types.py new file mode 100644 index 000000000..5ad74b42a --- /dev/null +++ b/apps/worker/app/services/document_agent/calibration/types.py @@ -0,0 +1,72 @@ +"""Calibration result types. + +Phase-1 produces only what Phase-2 cannot recompute: ``status``, per-regime +numbering ``kind`` + candidate ``offset`` (plus the anchor ``samples`` already +confirmed by vision), and one short ``notes`` reason. ``segments`` / +``no_toc_entry_indices`` / ``offset_status`` / per-regime ``notes`` are Phase-2 +outputs. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +# Failure classes recorded on ``CalibrationResult.failure_kind``, one per +# failure exit of Phase-1. +FAILURE_NO_OFFSET = "no_offset" +FAILURE_PAGE_COUNT_MISSING = "page_count_missing" +FAILURE_TOC_EMPTY = "toc_empty" + + +@dataclass +class CalibrationSample: + """A printed→physical anchor confirmed with ``inspect.pages``.""" + + title: str + physical: int | None = None + + +@dataclass +class CalibrationSegment: + """A contiguous leaf range that fully completed Phase-2 for one offset.""" + + offset: int + leaf_start: int + leaf_end: int + entry_indices: list[int] = field(default_factory=list) + status: str = "ok" + + +@dataclass +class CalibrationRegime: + kind: str + offset: int | None = None + # Optional membership override; empty → Phase-2 matches leaves by ``kind``. + entry_indices: list[int] = field(default_factory=list) + samples: list[CalibrationSample] = field(default_factory=list) + # Phase-2 outputs below; Phase-1 never fills these. + offset_status: str = "failed" + segments: list[CalibrationSegment] = field(default_factory=list) + no_toc_entry_indices: list[int] = field(default_factory=list) + notes: str = "" + + +@dataclass +class CalibrationResult: + status: str + regimes: list[CalibrationRegime] = field(default_factory=list) + offset: int | None = None + offset_status: str = "failed" + tool_calls: int = 0 + notes: str = "" + # Empty on success; otherwise one of the FAILURE_* constants. + failure_kind: str = "" + region_index: int | None = None + # Debug-only trail: one entry per scanned probe title. + scans: list[dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + diff --git a/apps/worker/app/services/document_agent/coarse_profile/__init__.py b/apps/worker/app/services/document_agent/coarse_profile/__init__.py new file mode 100644 index 000000000..4120640e7 --- /dev/null +++ b/apps/worker/app/services/document_agent/coarse_profile/__init__.py @@ -0,0 +1,5 @@ +"""One-shot VLM coarse document profiling.""" + +from app.services.document_agent.coarse_profile.classifier import CoarseProfiler + +__all__ = ["CoarseProfiler"] diff --git a/apps/worker/app/services/document_agent/planner/planner.py b/apps/worker/app/services/document_agent/coarse_profile/classifier.py similarity index 59% rename from apps/worker/app/services/document_agent/planner/planner.py rename to apps/worker/app/services/document_agent/coarse_profile/classifier.py index 48983f92f..d98beac96 100644 --- a/apps/worker/app/services/document_agent/planner/planner.py +++ b/apps/worker/app/services/document_agent/coarse_profile/classifier.py @@ -1,4 +1,9 @@ -"""Initial VLM profile planner for the document profile agent.""" +"""One-shot VLM coarse document classifier (PROFILE stage 0). + +Input: ``ToolContext`` with bootstrap page features / stats on the blackboard. +Output: ``DocumentProfile`` plus a ``ToolResult`` for tracing. +Does not choose tools or drive shard planning. +""" from __future__ import annotations @@ -11,16 +16,10 @@ from loguru import logger -from app.services.document_agent.manifest import ( - DocumentProfile, - ReflexionDecision, - ToolContext, - ToolResult, -) -from app.services.document_agent.planner.prompts import PLANNER_INSTRUCTIONS +from app.services.document_agent.coarse_profile.prompts import COARSE_PROFILE_INSTRUCTIONS +from app.services.document_agent.manifest import DocumentProfile, ToolContext, ToolResult from app.services.document_agent.visual import render_pages from app.services.document_parser.profiling.taxonomy import PdfRoutingCategory -from shared.utils.token_estimate import estimate_tokens PAGE_KIND_DEFINITIONS = { "normal": ( @@ -30,6 +29,9 @@ "landscape": "Landscape-oriented page, often wide tables, drawings, slides, or diagrams.", } +_COARSE_SAMPLE_CAP = 10 +_COARSE_SEGMENT_QUOTAS = (2, 2, 2) # front, middle, back + def _feature_rows(ctx: ToolContext, pages: list[int]) -> list[dict[str, Any]]: labels_by_page = {label.page: label for label in ctx.blackboard.page_labels} @@ -42,7 +44,6 @@ def _feature_rows(ctx: ToolContext, pages: list[int]) -> list[dict[str, Any]]: { "page": feature.page, "kind": label.kind if label else None, - "confidence": label.confidence if label else None, "raw_text_length": feature.raw_text_length, "text_density": feature.text_density, "orientation": feature.orientation, @@ -65,11 +66,6 @@ def _segment_sample(candidates: list[int], count: int) -> list[int]: return [candidates[round(index * step)] for index in range(count)] -# Coarse VLM budget: extrema first, then front/mid/back fill, hard cap 10. -_COARSE_SAMPLE_CAP = 10 -_COARSE_SEGMENT_QUOTAS = (2, 2, 2) # front, middle, back - - def _sample_pages( page_count: int, extrema_pages: list[int], @@ -87,14 +83,6 @@ def _sample_pages( 3. Optionally append ``random_extra`` uniform pages from the leftover pool (used when extrema were skipped because max text length is 0). 4. Hard truncate to ``_COARSE_SAMPLE_CAP``. - - Args: - page_count: Total number of pages. - extrema_pages: Pages with text extrema (min/max raw_text_length / - text_density). Low-text extrema already surface chart/asset pages. - random_extra: Extra pages to draw uniformly from pages not already - selected. - rng: Optional RNG for deterministic tests. """ if page_count <= 0: return [] @@ -144,10 +132,11 @@ def _parse_margin_ratio(value: Any) -> float | None: return None -def _parse_profile_and_decision(raw: str) -> tuple[DocumentProfile, ReflexionDecision]: +def _parse_profile(raw: str) -> DocumentProfile: + """Parse coarse-profile VLM JSON into ``DocumentProfile``.""" data = json.loads(raw) if not isinstance(data, dict): - raise ValueError("planner output must be a JSON object") + raise ValueError("coarse profile output must be a JSON object") category = " ".join(str(data.get("category") or "unknown document").split()[:5]) routing_category = PdfRoutingCategory.normalize( data.get("routing_category") or data.get("category") @@ -164,7 +153,7 @@ def _parse_profile_and_decision(raw: str) -> tuple[DocumentProfile, ReflexionDec if header_y is not None and footer_y is not None and header_y >= footer_y: header_y = None footer_y = None - profile = DocumentProfile( + return DocumentProfile( is_scanned=is_scanned, category=category or "unknown document", routing_category=routing_category, @@ -173,47 +162,17 @@ def _parse_profile_and_decision(raw: str) -> tuple[DocumentProfile, ReflexionDec header_y=header_y, footer_y=footer_y, ) - next_action = str(data.get("next_action") or "ready_to_shard").strip().lower() - # Legacy models may still emit verdict_now; that is not a planner finish - # signal — fall through to ready_to_shard so the executor owns success/abort. - # Legacy inspect_more is ignored the same way (tool removed). - if next_action in {"verdict_now", "inspect_more"}: - next_action = "ready_to_shard" - tool_name: str | None = None - tool_args: dict[str, Any] = {} - if next_action == "grep_text" and not profile.is_scanned: - query = str(data.get("grep_query") or "").strip() - if query: - tool_name = "grep.text" - tool_args = {"query": query, "max_results": 20} - if tool_name: - return profile, ReflexionDecision( - action="tool_call", - rationale=profile.rationale, - tool_name=tool_name, - tool_args=tool_args, - ) - return profile, ReflexionDecision( - action="tool_call", - rationale=profile.rationale, - tool_name="propose.shard_plan", - tool_args={}, - ) -class ProfilePlanner: - """One-shot VLM planner that profiles the document and proposes the first action.""" +class CoarseProfiler: + """One-shot VLM classifier: pages + features → ``DocumentProfile``.""" def __init__(self, ctx: ToolContext) -> None: self.ctx = ctx - def propose(self) -> tuple[DocumentProfile, ReflexionDecision, ToolResult]: + def classify(self) -> tuple[DocumentProfile, ToolResult]: start = time.monotonic() - model = ( - self.ctx.settings.get("planner_model") - or self.ctx.settings.get("vlm_model") - or os.environ.get("IMAGE_MODEL") - ) + model = self.ctx.settings.get("vlm_model") or os.environ.get("IMAGE_MODEL") text_max = float( ( ((self.ctx.blackboard.doc_stats or {}).get("raw_text_length") or {}).get( @@ -241,27 +200,22 @@ def propose(self) -> tuple[DocumentProfile, ReflexionDecision, ToolResult]: is_scanned=False, category="unknown document", routing_category=PdfRoutingCategory.GENERIC.value, - rationale="No planner model configured.", - ) - decision = ReflexionDecision( - action="tool_call", - rationale=profile.rationale, - tool_name="propose.shard_plan", - tool_args={}, + rationale="No VLM model configured.", ) - return profile, decision, ToolResult( + return profile, ToolResult( status="ok", payload={"source": "deterministic", "sampled_pages": pages}, latency_ms=int((time.monotonic() - start) * 1000), - warnings=["No planner model configured; using conservative profile."], + warnings=["No VLM model configured; using conservative profile."], input_summary={"page_count": self.ctx.blackboard.page_count}, - output_summary={"profile": profile.to_dict(), "decision": decision.to_dict()}, + output_summary={"profile": profile.to_dict()}, ) + pngs = render_pages( self.ctx, pages, - folder_name="planner_pages", - prefix="planner", + folder_name="coarse_profile_pages", + prefix="coarse", timeout=180, ) feature_summary = _feature_rows(self.ctx, pages) @@ -279,21 +233,11 @@ def propose(self) -> tuple[DocumentProfile, ReflexionDecision, ToolResult]: [], ), "sampled_page_features": feature_summary, - "available_actions": [ - "grep.text", - "propose.shard_plan", - "validate.anatomy_map", - "verdict", - ], } - prompt_text = PLANNER_INSTRUCTIONS + "\nPayload:\n" + json.dumps( + prompt_text = COARSE_PROFILE_INSTRUCTIONS + "\nPayload:\n" + json.dumps( payload, ensure_ascii=False, ) - prompt_tokens_est = estimate_tokens(prompt_text) + len(pngs) * 800 - stage = "coarse_planner" - if not self.ctx.budget.try_reserve("visual", prompt_tokens_est, stage=stage): - raise RuntimeError("Insufficient visual budget for profile planning.") content_parts: list[dict[str, Any]] = [{"type": "text", "text": prompt_text}] for item in pngs: @@ -310,45 +254,33 @@ def propose(self) -> tuple[DocumentProfile, ReflexionDecision, ToolResult]: } ) except Exception as exc: - logger.warning("[document_agent] planner png attach failed: {}", exc) + logger.warning( + "[document_agent] coarse profile png attach failed: {}", exc + ) - try: - from shared.services.ai.llm_overrides import get_vision_client + from shared.services.ai.llm_overrides import get_vision_client - client, model = get_vision_client(requested_model=model) - raw, usage = client.chat_completion_with_usage( - messages=cast(Any, [{"role": "user", "content": content_parts}]), - model=model, - temperature=0.0, - max_tokens=1800, - response_format={"type": "json_object"}, - usage_task="document_agent.coarse_profile", - ) - self.ctx.budget.commit( - "visual", - actual=usage.get("total_tokens", prompt_tokens_est), - est=prompt_tokens_est, - stage=stage, - ) - profile, decision = _parse_profile_and_decision(raw) - return profile, decision, ToolResult( - status="ok", - payload={ - "source": "llm", - "sampled_pages": pages, - "first_action": decision.tool_name, - }, - latency_ms=int((time.monotonic() - start) * 1000), - tokens_used=usage.get("total_tokens", 0), - input_summary=payload, - output_summary={"profile": profile.to_dict(), "decision": decision.to_dict()}, - debug={ - "prompt_text": prompt_text, - "sampled_pages": pages, - "sampled_pngs": pngs, - "raw_response": raw, - }, - ) - except Exception: - self.ctx.budget.refund("visual", est=prompt_tokens_est, stage=stage) - raise + client, model = get_vision_client(requested_model=model) + raw, usage = client.chat_completion_with_usage( + messages=cast(Any, [{"role": "user", "content": content_parts}]), + model=model, + temperature=0.0, + max_tokens=1800, + response_format={"type": "json_object"}, + usage_task="document_agent.coarse_profile", + ) + profile = _parse_profile(raw) + return profile, ToolResult( + status="ok", + payload={"source": "llm", "sampled_pages": pages}, + latency_ms=int((time.monotonic() - start) * 1000), + tokens_used=usage.get("total_tokens", 0), + input_summary=payload, + output_summary={"profile": profile.to_dict()}, + debug={ + "prompt_text": prompt_text, + "sampled_pages": pages, + "sampled_pngs": pngs, + "raw_response": raw, + }, + ) diff --git a/apps/worker/app/services/document_agent/planner/prompts.py b/apps/worker/app/services/document_agent/coarse_profile/prompts.py similarity index 56% rename from apps/worker/app/services/document_agent/planner/prompts.py rename to apps/worker/app/services/document_agent/coarse_profile/prompts.py index 6772e59f5..74e3acfd8 100644 --- a/apps/worker/app/services/document_agent/planner/prompts.py +++ b/apps/worker/app/services/document_agent/coarse_profile/prompts.py @@ -1,10 +1,10 @@ -"""Prompts for the document profile planner.""" +"""Prompts for one-shot coarse document profiling.""" -PLANNER_INSTRUCTIONS = ( - "You are a document profile agent. Use page-feature statistics " +COARSE_PROFILE_INSTRUCTIONS = ( + "You are a document profile classifier. Use page-feature statistics " "and the provided page screenshots to classify the PDF. " "Return strict JSON only with keys: is_scanned, category, routing_category, " - "language, rationale, header_y, footer_y, next_action, grep_query. " + "language, rationale, header_y, footer_y. " "category is a concise semantic document type in at most 5 English words. " "routing_category must be one of atlas, scanned, slides, generic. " "Set routing_category=atlas only when pages are primarily drawing/detail " @@ -15,12 +15,7 @@ "you observe (largest y) when any header is present, otherwise null; " "footer_y is the highest footer line you observe (smallest y) when any " "footer is present, otherwise null. When both are set, require " - "header_y < footer_y. " - "next_action must be one of grep_text, ready_to_shard. " - "Use grep_text only for native PDFs when a global text search would clarify " - "structure. Use ready_to_shard when evidence is sufficient to propose shards. " - "Do not finish or abort the profile run from next_action; the executor owns " - "success/abort via the verdict tool. Do not output a fixed step plan." + "header_y < footer_y." ) -__all__ = ["PLANNER_INSTRUCTIONS"] +__all__ = ["COARSE_PROFILE_INSTRUCTIONS"] diff --git a/apps/worker/app/services/document_agent/coordinator.py b/apps/worker/app/services/document_agent/coordinator.py index 04beae484..f9a43b15d 100644 --- a/apps/worker/app/services/document_agent/coordinator.py +++ b/apps/worker/app/services/document_agent/coordinator.py @@ -1,8 +1,7 @@ -"""ReAct-style coordinator for the document profile agent.""" +"""Coordinator for the document profile workflow.""" from __future__ import annotations -import os from typing import Any from loguru import logger @@ -13,9 +12,8 @@ probe_page_assets, probe_page_features, ) -from app.services.document_agent.budget import BudgetTracker, StageEnvelope -from app.services.document_agent.executor import ReActExecutor from app.services.document_agent.manifest import ( + ProfileVerdict, DocumentProfile, PageAnatomyMap, TocResult, @@ -24,13 +22,10 @@ ) from app.services.document_agent.pdf_text import read_page_texts from app.services.document_agent.persist import build_anatomy_map, persist_anatomy_map -from app.services.document_agent.planner import ProfilePlanner +from app.services.document_agent.coarse_profile import CoarseProfiler from app.services.document_agent.registry import REGISTRY -from app.services.document_agent.state import AgentBlackboard, DocumentAgentState +from app.services.document_agent.state import ProfileBlackboard, ProfileState from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring -from app.services.document_agent.structure.toc_link_enrichment import ( - enrich_toc_hierarchies_with_links, -) from app.services.document_agent import tools as _registered_tools # noqa: F401 from app.services.document_agent.trace import ParseRunRecorder from app.services.document_agent.validators import single_shard_plan @@ -47,44 +42,8 @@ def __init__( model: str | None = None, settings: dict[str, Any] | None = None, ) -> None: - self.state = DocumentAgentState.INIT - self.blackboard = AgentBlackboard() - self.budget = BudgetTracker( - plan_budget=int(os.environ.get("PARSE_AGENT_PLAN_BUDGET", "50000")), - visual_budget=int(os.environ.get("PARSE_AGENT_VISUAL_BUDGET", "120000")), - visual_stage_envelopes={ - "toc_confirm": StageEnvelope( - min_guarantee=int( - os.environ.get("PARSE_AGENT_TOC_CONFIRM_MIN_BUDGET", "8000") - ), - cap=int(os.environ.get("PARSE_AGENT_TOC_CONFIRM_CAP", "24000")), - ), - "coarse_planner": StageEnvelope( - min_guarantee=int( - os.environ.get("PARSE_AGENT_COARSE_PLANNER_MIN_BUDGET", "12000") - ), - cap=int(os.environ.get("PARSE_AGENT_COARSE_PLANNER_CAP", "36000")), - ), - "structural_react": StageEnvelope( - min_guarantee=int( - os.environ.get("PARSE_AGENT_STRUCTURAL_REACT_MIN_BUDGET", "24000") - ), - cap=int(os.environ.get("PARSE_AGENT_STRUCTURAL_REACT_CAP", "64000")), - ), - "calibration": StageEnvelope( - min_guarantee=int( - os.environ.get("PARSE_AGENT_CALIBRATION_MIN_BUDGET", "12000") - ), - cap=int(os.environ.get("PARSE_AGENT_CALIBRATION_CAP", "40000")), - ), - "page_tagging": StageEnvelope( - min_guarantee=int( - os.environ.get("PARSE_AGENT_PAGE_TAGGING_MIN_BUDGET", "0") - ), - cap=int(os.environ.get("PARSE_AGENT_PAGE_TAGGING_CAP", "0")) or None, - ), - }, - ) + self.state = ProfileState.INIT + self.blackboard = ProfileBlackboard() effective_settings = settings or {} if model: effective_settings["model"] = model @@ -92,7 +51,6 @@ def __init__( pdf_path=pdf_path, job_id=job_id, blackboard=self.blackboard, - budget=self.budget, trace=None, output_dir=output_dir, settings=effective_settings, @@ -100,7 +58,7 @@ def __init__( self.trace = ParseRunRecorder(job_id=job_id, db=db) self.ctx.trace = self.trace self.round_index = 0 - self._planner_cache: tuple[DocumentProfile, Any, ToolResult] | None = None + self._coarse_profile_cache: DocumentProfile | None = None def run_coarse(self) -> DocumentProfile: try: @@ -126,12 +84,10 @@ def run_lightweight_anatomy( raise def _run_coarse(self) -> DocumentProfile: - self.state = DocumentAgentState.RUNNING + self.state = ProfileState.RUNNING if not self.blackboard.page_features: self._run_bootstrap() - profile, _initial_decision, _planner_result = self._propose_profile( - actor="planner:coarse" - ) + profile = self._ensure_coarse_profile(actor="coarse_profile") self._run_text_scan() # Asset coarse probe is independent of TOC; run it before TOC so # PROFILE / debug Stage-0 share the same order as later anatomy/shard @@ -147,7 +103,7 @@ def _run_coarse(self) -> DocumentProfile: return profile def _run_structural(self, *, skip_shard_plan: bool = False) -> PageAnatomyMap: - self.state = DocumentAgentState.RUNNING + self.state = ProfileState.RUNNING if not self.blackboard.page_features: self._run_bootstrap() # Prefer assets before TOC so cold structural matches coarse order. @@ -158,24 +114,13 @@ def _run_structural(self, *, skip_shard_plan: bool = False) -> PageAnatomyMap: self._ensure_toc_profile(strict=True) else: self._ensure_disabled_toc_placeholder() - profile, initial_decision, _planner_result = self._propose_profile( - actor="planner" - ) + self._ensure_coarse_profile(actor="coarse_profile") if skip_shard_plan: # Page-memory oversized path never consumes shard_plan; only # build_anatomy_map's invariant needs a non-empty plan. self._apply_single_shard_placeholder() else: - executor_result = ReActExecutor( - self.ctx, - registry=REGISTRY, - max_rounds=int(self.ctx.settings.get("max_rounds", 30)), - initial_decision=initial_decision, - ).run() - if executor_result.verdict.status != "success": - raise RuntimeError( - f"profile aborted: {executor_result.verdict.rationale}" - ) + self._finalize_shard_plan() anatomy = build_anatomy_map(self.ctx) self._persist_ready_anatomy(anatomy) return anatomy @@ -183,7 +128,7 @@ def _run_structural(self, *, skip_shard_plan: bool = False) -> PageAnatomyMap: def _run_lightweight_anatomy( self, *, skip_shard_plan: bool = False ) -> PageAnatomyMap: - self.state = DocumentAgentState.RUNNING + self.state = ProfileState.RUNNING if not self.blackboard.page_features: self._run_bootstrap() # Same relative order as coarse: assets before any TOC placeholder. @@ -204,18 +149,7 @@ def _run_lightweight_anatomy( # (kept for chunk-track oversized MinerU sharding). self._apply_single_shard_placeholder() else: - result = REGISTRY.dispatch("propose.shard_plan", self.ctx, {}) - self.trace.record_step( - round_index=self.round_index, - actor="anatomy:propose.shard_plan", - action_type="anatomy", - result=result, - tool_name="propose.shard_plan", - tool_args={}, - ) - if result.status not in {"ok", "invalid"}: - raise RuntimeError(result.error or "propose.shard_plan failed") - self.round_index += 1 + self._dispatch_anatomy_tool(tool_name="propose.shard_plan") anatomy = build_anatomy_map(self.ctx) self._persist_ready_anatomy(anatomy) return anatomy @@ -223,6 +157,55 @@ def _run_lightweight_anatomy( def _apply_single_shard_placeholder(self) -> None: self.blackboard.shard_plan = single_shard_plan(self.blackboard.page_count) + def _dispatch_anatomy_tool( + self, + *, + tool_name: str, + tool_args: dict[str, Any] | None = None, + ) -> ToolResult: + args = dict(tool_args or {}) + result = REGISTRY.dispatch(tool_name, self.ctx, args) + self.trace.record_step( + round_index=self.round_index, + actor=f"anatomy:{tool_name}", + action_type="anatomy", + result=result, + tool_name=tool_name, + tool_args=args, + ) + if result.status not in {"ok", "invalid"}: + raise RuntimeError(result.error or f"{tool_name} failed") + self.round_index += 1 + return result + + def _finalize_shard_plan(self) -> None: + """Deterministic propose → validate → verdict. + + Cut logic stays inside ``propose.shard_plan``. Invalid validation aborts; + do not disguise a single-shard rewrite as a successful propose. + """ + if self.blackboard.shard_plan is None: + self._dispatch_anatomy_tool(tool_name="propose.shard_plan") + if not self.blackboard.validation_report: + self._dispatch_anatomy_tool(tool_name="validate.anatomy_map") + if (self.blackboard.validation_report or {}).get("valid") is True: + self._dispatch_anatomy_tool( + tool_name="verdict", + tool_args={ + "status": "success", + "rationale": "Validation succeeded; finishing profile run.", + }, + ) + return + + self.blackboard.verdict = ProfileVerdict( + status="abort", + rationale="Shard plan validation failed.", + ) + raise RuntimeError( + f"profile aborted: {self.blackboard.verdict.rationale}" + ) + def _persist_ready_anatomy(self, anatomy: PageAnatomyMap) -> None: persist_result = persist_anatomy_map(self.ctx, {}) self.trace.record_step( @@ -233,7 +216,7 @@ def _persist_ready_anatomy(self, anatomy: PageAnatomyMap) -> None: tool_name="persist.anatomy_map", tool_args={}, ) - self.state = DocumentAgentState.READY + self.state = ProfileState.READY self.trace.write_trace_artifact( self.ctx.output_dir, final_status="ready", @@ -246,11 +229,11 @@ def _persist_ready_anatomy(self, anatomy: PageAnatomyMap) -> None: def _record_failure(self, exc: Exception) -> None: logger.error(f"[document_agent] profile failed: {exc}") - self.state = DocumentAgentState.FAILED + self.state = ProfileState.FAILED self.trace.write_trace_artifact( self.ctx.output_dir, final_status="failed", - summary={"error": str(exc), "budget": self.ctx.budget.snapshot()}, + summary={"error": str(exc)}, ) self.trace.flush(final_status="failed", summary={"error": str(exc)}) @@ -309,7 +292,7 @@ def _toc_profile_enabled(self) -> bool: def _run_text_scan(self) -> None: profile = self.blackboard.document_profile if profile is None: - raise RuntimeError("document_profile missing; run planner first") + raise RuntimeError("document_profile missing; run coarse profile first") page_count = int(self.blackboard.page_count or 0) pages = list(range(1, page_count + 1)) if not pages: @@ -370,18 +353,13 @@ def _ensure_toc_profile(self, *, strict: bool) -> None: self._run_toc_extraction_pipeline() except Exception as exc: logger.warning( - "[document_agent] TOC profiling failed, " - "degrading to empty TOC: {}", + "[document_agent] TOC profiling failed: {}", exc, ) - self.blackboard.toc_result = TocResult( - method="none", - notes=f"degraded: {type(exc).__name__}: {exc}", - failure_kind="degraded", - ) + self.blackboard.toc_result = None self.blackboard.toc_hierarchies = None self._clear_toc_anchor_state() - return + raise if self.blackboard.toc_result is None: self.blackboard.toc_result = TocResult( @@ -389,24 +367,24 @@ def _ensure_toc_profile(self, *, strict: bool) -> None: notes="TOC extraction completed without a result", ) - def _propose_profile(self, *, actor: str) -> tuple[DocumentProfile, Any, ToolResult]: - if self._planner_cache is not None: - return self._planner_cache + def _ensure_coarse_profile(self, *, actor: str) -> DocumentProfile: + """Run one-shot coarse VLM classification once; cache the profile.""" + if self._coarse_profile_cache is not None: + return self._coarse_profile_cache - profile, initial_decision, planner_result = ProfilePlanner(self.ctx).propose() + profile, result = CoarseProfiler(self.ctx).classify() self.blackboard.document_profile = profile - self.blackboard.global_signals["document_profile"] = profile.to_dict() self.trace.record_step( round_index=self.round_index, actor=actor, - action_type="plan", - result=planner_result, + action_type="coarse_profile", + result=result, tool_name=None, tool_args={}, ) self.round_index += 1 - self._planner_cache = (profile, initial_decision, planner_result) - return self._planner_cache + self._coarse_profile_cache = profile + return profile def _dispatch_profile_tool(self, *, tool_name: str, actor: str) -> ToolResult: result = REGISTRY.dispatch(tool_name, self.ctx, {}) @@ -424,20 +402,22 @@ def _dispatch_profile_tool(self, *, tool_name: str, actor: str) -> ToolResult: return result def _clear_toc_anchor_state(self) -> None: - self.blackboard.toc_page_offset = None self.blackboard.skeleton_anchor = None self.blackboard.skeleton_nodes = None self.blackboard.pending_skeleton_anchors = [] def _run_toc_extraction_pipeline(self) -> None: - for tool_name in ("find.toc_anchor_pages", "extract.toc_with_boundaries"): + for tool_name in ( + "find.toc_anchor_pages", + "probe.outline", + "extract.toc_with_boundaries", + ): self._dispatch_profile_tool( tool_name=tool_name, actor=f"toc:{tool_name}", ) - self._attach_toc_page_links() if self.ctx.settings.get("skip_toc_anchoring"): - # Debug Stage-1: stop after TOC extract + link attach. + # Debug Stage-1: stop after TOC extract. self._clear_toc_anchor_state() logger.info( "[document_agent] skip_toc_anchoring=True; " @@ -446,28 +426,3 @@ def _run_toc_extraction_pipeline(self) -> None: return run_toc_anchoring(self.ctx) - def _attach_toc_page_links(self) -> None: - """Attach TOC-page hyperlinks onto VLM entries before calibration.""" - hierarchies = list(self.blackboard.toc_hierarchies or []) - if not hierarchies: - return - try: - enriched, stats = enrich_toc_hierarchies_with_links( - pdf_path=self.ctx.pdf_path, - toc_hierarchies=hierarchies, - ) - except Exception as exc: - logger.warning( - "[document_agent] TOC link attach failed, " - "continuing without links: {}", - exc, - ) - return - self.blackboard.toc_hierarchies = enriched - logger.info( - "[document_agent] TOC link attach: matched={}/{} skipped_no_links={}", - stats.entries_matched, - stats.entries_total, - stats.skipped_no_links, - ) - diff --git a/apps/worker/app/services/document_agent/executor/__init__.py b/apps/worker/app/services/document_agent/executor/__init__.py deleted file mode 100644 index e5656b258..000000000 --- a/apps/worker/app/services/document_agent/executor/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""ReAct executor for the document profile agent.""" - -from app.services.document_agent.executor.react_loop import ( - ExecutorResult, - ReActExecutor, - _parse_decision, -) - -__all__ = [ - "ExecutorResult", - "ReActExecutor", - "_parse_decision", -] diff --git a/apps/worker/app/services/document_agent/executor/prompts.py b/apps/worker/app/services/document_agent/executor/prompts.py deleted file mode 100644 index a19cec544..000000000 --- a/apps/worker/app/services/document_agent/executor/prompts.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Prompts for executor reflexion.""" - -REFLEXION_INSTRUCTIONS = ( - "You are the executor of a document profiling agent. Decide the next tool " - "call from the blackboard facts and available tools. Return strict JSON with " - "keys: action (must be tool_call), rationale, tool_name, tool_args. " - "Use grep.text when native-PDF text evidence is needed, propose.shard_plan " - "when evidence is sufficient to shard, validate.anatomy_map after a shard " - "plan exists, and the verdict tool to finish: verdict(status=success) only " - "after validation succeeds, or verdict(status=abort, rationale=...) only " - "when the document cannot be profiled. Do not invent other finish actions." -) - -__all__ = ["REFLEXION_INSTRUCTIONS"] diff --git a/apps/worker/app/services/document_agent/executor/react_loop.py b/apps/worker/app/services/document_agent/executor/react_loop.py deleted file mode 100644 index 994f6390c..000000000 --- a/apps/worker/app/services/document_agent/executor/react_loop.py +++ /dev/null @@ -1,340 +0,0 @@ -"""ReAct-style executor for the document profile agent.""" - -from __future__ import annotations - -import json -import time -from dataclasses import dataclass -from typing import Any - -from app.services.document_agent.manifest import ( - AgentVerdict, - ReflexionDecision, - ToolContext, - ToolResult, -) -from app.services.document_agent.executor.prompts import REFLEXION_INSTRUCTIONS -from app.services.document_agent.registry import ToolRegistry -from shared.utils.token_estimate import estimate_tokens - - -@dataclass -class ExecutorResult: - verdict: AgentVerdict - rounds: int - - -def _compact_blackboard(ctx: ToolContext) -> dict[str, Any]: - return { - "page_count": ctx.blackboard.page_count, - "page_kind_counts": ctx.blackboard.global_signals.get("page_kind_counts", {}), - "doc_stats": ctx.blackboard.doc_stats, - "extrema_pages": ctx.blackboard.extrema_pages, - "document_profile": ctx.blackboard.document_profile.to_dict() - if ctx.blackboard.document_profile - else None, - "toc_anchor_pages": [anchor.page for anchor in ctx.blackboard.toc_anchor_pages], - "toc_pages": ( - ctx.blackboard.toc_result.toc_pages if ctx.blackboard.toc_result else [] - ), - "toc_hierarchies_count": len(ctx.blackboard.toc_hierarchies or []), - "shard_plan": ctx.blackboard.shard_plan.to_dict() - if ctx.blackboard.shard_plan - else None, - "validation_report": ctx.blackboard.validation_report, - "verdict": ctx.blackboard.verdict.to_dict() - if ctx.blackboard.verdict - else None, - "grep_history": ctx.blackboard.global_signals.get("grep_history", [])[-3:], - "budget": ctx.budget.snapshot(), - } - - -def _coerce_legacy_finish(data: dict[str, Any]) -> ReflexionDecision: - """Map obsolete ``action=verdict_now`` into a real tool call. - - Finish belongs exclusively to the ``verdict`` tool. A bare ``verdict_now`` - without an explicit status is treated as ready-to-shard, never as abort. - """ - rationale = str(data.get("rationale") or "") - raw_verdict = data.get("verdict") - if isinstance(raw_verdict, dict): - status = str(raw_verdict.get("status") or "").strip().lower() - if status in {"success", "abort"}: - return ReflexionDecision( - action="tool_call", - rationale=rationale, - tool_name="verdict", - tool_args={ - "status": status, - "rationale": str( - raw_verdict.get("rationale") or rationale or status - ), - }, - ) - return ReflexionDecision( - action="tool_call", - rationale=rationale or "Legacy verdict_now without status; propose shard plan.", - tool_name="propose.shard_plan", - tool_args={}, - ) - - -def _parse_decision(raw: str) -> ReflexionDecision: - data = json.loads(raw) - action = str(data.get("action") or "tool_call").strip().lower() - if action == "verdict_now": - return _coerce_legacy_finish(data if isinstance(data, dict) else {}) - if action != "tool_call": - action = "tool_call" - return ReflexionDecision( - action="tool_call", - rationale=str(data.get("rationale") or ""), - tool_name=data.get("tool_name"), - tool_args=dict(data.get("tool_args") or {}), - verdict=None, - ) - - -class ReActExecutor: - def __init__( - self, - ctx: ToolContext, - *, - registry: ToolRegistry, - max_rounds: int = 30, - initial_decision: ReflexionDecision | None = None, - ) -> None: - self.ctx = ctx - self.registry = registry - self.max_rounds = max_rounds - self._initial_decision = initial_decision - - def run(self) -> ExecutorResult: - for round_index in range(self.max_rounds): - pending_recovery_verdict: AgentVerdict | None = None - decision, result = self._next_decision(round_index) - if decision.action != "tool_call": - # Defensive: only tool_call is a legal executor step. - decision = ReflexionDecision( - action="tool_call", - rationale=decision.rationale - or "Non-tool executor action coerced to propose.shard_plan.", - tool_name="propose.shard_plan", - tool_args={}, - ) - self.ctx.blackboard.global_signals.setdefault("reflexion_decisions", []).append( - decision.to_dict() - ) - if self.ctx.trace: - self.ctx.trace.record_step( - round_index=round_index, - actor=f"executor:r{round_index}", - action_type="reflexion", - result=result, - tool_name=decision.tool_name, - tool_args=decision.tool_args, - ) - - tool_name, tool_args = self._resolve_tool_call(decision) - if tool_name == "verdict" and str(tool_args.get("status") or "") == "success": - if not ( - self.ctx.blackboard.validation_report - and self.ctx.blackboard.validation_report.get("valid") is True - ): - decision = ReflexionDecision( - action="tool_call", - rationale=( - "Validate the anatomy map before accepting a success verdict." - ), - tool_name="validate.anatomy_map", - tool_args={}, - ) - tool_name, tool_args = self._resolve_tool_call(decision) - - if not tool_name: - verdict = AgentVerdict( - status="abort", - rationale="Executor did not choose a tool.", - ) - self.ctx.blackboard.verdict = verdict - return ExecutorResult(verdict=verdict, rounds=round_index + 1) - - tool_result = self.registry.dispatch(tool_name, self.ctx, tool_args) - if self.ctx.trace: - self.ctx.trace.record_step( - round_index=round_index, - actor=f"tool:{tool_name}", - action_type="tool_call", - result=tool_result, - tool_name=tool_name, - tool_args=tool_args, - ) - self.ctx.blackboard.step_history.append( - { - "round": round_index, - "tool_name": tool_name, - "tool_args": tool_args, - "status": tool_result.status, - "error": tool_result.error, - } - ) - if tool_result.status == "error": - pending_recovery_verdict = AgentVerdict( - status="abort", - rationale=tool_result.error or f"{tool_name} failed", - ) - elif tool_result.status == "precondition_unmet": - pending_recovery_verdict = AgentVerdict( - status="abort", - rationale=tool_result.error or f"{tool_name} precondition unmet", - ) - - if self.ctx.blackboard.verdict is not None: - return ExecutorResult( - verdict=self.ctx.blackboard.verdict, - rounds=round_index + 1, - ) - if pending_recovery_verdict is not None and self._is_deterministic_mode(): - self.ctx.blackboard.verdict = pending_recovery_verdict - return ExecutorResult( - verdict=pending_recovery_verdict, - rounds=round_index + 1, - ) - - verdict = AgentVerdict(status="abort", rationale="Maximum executor rounds reached.") - self.ctx.blackboard.verdict = verdict - return ExecutorResult(verdict=verdict, rounds=self.max_rounds) - - def _resolve_tool_call( - self, - decision: ReflexionDecision, - ) -> tuple[str | None, dict[str, Any]]: - if decision.action == "tool_call" and decision.tool_name: - return decision.tool_name, decision.tool_args - return None, {} - - def _is_deterministic_mode(self) -> bool: - return not (self.ctx.settings.get("executor_model") or self.ctx.settings.get("model")) - - def _next_decision(self, round_index: int) -> tuple[ReflexionDecision, ToolResult]: - if round_index == 0 and self._initial_decision is not None: - decision = self._initial_decision - if decision.action != "tool_call": - decision = ReflexionDecision( - action="tool_call", - rationale=decision.rationale - or "Initial non-tool decision coerced to propose.shard_plan.", - tool_name="propose.shard_plan", - tool_args={}, - ) - elif not decision.tool_name: - decision = ReflexionDecision( - action="tool_call", - rationale=decision.rationale - or "Initial decision missing tool; propose shard plan.", - tool_name="propose.shard_plan", - tool_args={}, - ) - return decision, ToolResult(status="ok", payload=decision.to_dict()) - model = self.ctx.settings.get("executor_model") or self.ctx.settings.get("model") - if not model: - decision = self._deterministic_decision() - return decision, ToolResult(status="ok", payload=decision.to_dict()) - - payload = { - "blackboard": _compact_blackboard(self.ctx), - "history_tail": self.ctx.blackboard.step_history[-6:], - "available_tools": self.registry.openai_specs(self.ctx.blackboard), - "round_index": round_index, - } - prompt = REFLEXION_INSTRUCTIONS + "\nPayload:\n" + json.dumps( - payload, - ensure_ascii=False, - ) - est = estimate_tokens(prompt) - if not self.ctx.budget.try_reserve("plan", est): - decision = ReflexionDecision( - action="tool_call", - rationale="Planner budget exhausted.", - tool_name="verdict", - tool_args={ - "status": "abort", - "rationale": "Planner budget exhausted.", - }, - ) - return decision, ToolResult( - status="ok", - payload=decision.to_dict(), - input_summary=payload, - ) - start = time.monotonic() - try: - from shared.services.ai.llm_overrides import get_text_client - - client, model = get_text_client(requested_model=model) - raw, usage = client.chat_completion_with_usage( - messages=[{"role": "user", "content": prompt}], - model=model, - temperature=0.0, - max_tokens=1200, - response_format={"type": "json_object"}, - usage_task="document_agent.react_loop", - ) - self.ctx.budget.commit( - "plan", - actual=usage.get("total_tokens", est), - est=est, - ) - decision = _parse_decision(raw) - return decision, ToolResult( - status="ok", - payload=decision.to_dict(), - latency_ms=int((time.monotonic() - start) * 1000), - tokens_used=usage.get("total_tokens", 0), - input_summary=payload, - debug={"prompt_text": prompt, "raw_response": raw}, - ) - except Exception: - self.ctx.budget.refund("plan", est=est) - raise - - def _deterministic_decision(self) -> ReflexionDecision: - if self.ctx.blackboard.shard_plan is None: - return ReflexionDecision( - action="tool_call", - rationale="Create a shard plan.", - tool_name="propose.shard_plan", - tool_args={}, - ) - if not self.ctx.blackboard.validation_report: - return ReflexionDecision( - action="tool_call", - rationale="Validate the current shard plan.", - tool_name="validate.anatomy_map", - tool_args={}, - ) - if self.ctx.blackboard.validation_report.get("valid") is True: - return ReflexionDecision( - action="tool_call", - rationale="Validation succeeded; finish profile run.", - tool_name="verdict", - tool_args={ - "status": "success", - "rationale": "Validation succeeded; finishing profile run.", - }, - ) - # Validation failed: fallback to single shard instead of aborting. - # Clear the invalid plan and re-propose as a single shard. - from app.services.document_agent.tools.propose_shard_plan import single_shard_plan - - self.ctx.blackboard.shard_plan = single_shard_plan( - self.ctx.blackboard.page_count - ) - self.ctx.blackboard.validation_report = None - return ReflexionDecision( - action="tool_call", - rationale="Validation failed; falling back to single shard plan.", - tool_name="validate.anatomy_map", - tool_args={}, - ) diff --git a/apps/worker/app/services/document_agent/manifest.py b/apps/worker/app/services/document_agent/manifest.py index b3517b50b..659e40cf9 100644 --- a/apps/worker/app/services/document_agent/manifest.py +++ b/apps/worker/app/services/document_agent/manifest.py @@ -1,4 +1,4 @@ -"""Contracts for the hierarchy-first document profile agent.""" +"""Contracts for the hierarchy-first document profile workflow.""" from __future__ import annotations @@ -10,10 +10,7 @@ PageKind = Literal["normal", "landscape"] TocFailureKind = Literal["none", "confirm_failed", "rejected_all", "degraded"] -# Executor loop steps are always tool calls. Profile success/abort is owned -# exclusively by the ``verdict`` tool (AgentVerdict.status), not by a separate -# ReflexionAction shortcut. -ReflexionAction = Literal["tool_call"] +# Profile success/abort is owned exclusively by the ``verdict`` tool. VerdictStatus = Literal["success", "abort"] @@ -43,7 +40,6 @@ def to_dict(self) -> dict[str, Any]: class PageLabel: page: int kind: PageKind - confidence: float evidence: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: @@ -67,7 +63,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass -class AgentVerdict: +class ProfileVerdict: status: VerdictStatus rationale: str @@ -75,43 +71,13 @@ def to_dict(self) -> dict[str, Any]: return asdict(self) -@dataclass -class ReflexionDecision: - action: ReflexionAction - rationale: str - tool_name: str | None = None - tool_args: dict[str, Any] = field(default_factory=dict) - verdict: AgentVerdict | None = None - - def to_dict(self) -> dict[str, Any]: - return { - "action": self.action, - "rationale": self.rationale, - "tool_name": self.tool_name, - "tool_args": dict(self.tool_args), - "verdict": self.verdict.to_dict() if self.verdict else None, - } - - -@dataclass -class TocCandidate: - title: str - normalized_title: str - source_page: int - line_index: int - numbering: str = "" - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - @dataclass class TocAnchorPage: """A candidate TOC start page identified by keyword scan, pending VLM confirmation.""" page: int # 1-based page number png_path: str # local PNG path for VLM inspection - source: Literal["page_label", "text_scan", "visual_scan"] # how this anchor was discovered + source: Literal["text_scan"] = "text_scan" def to_dict(self) -> dict[str, Any]: return asdict(self) @@ -121,7 +87,6 @@ def to_dict(self) -> dict[str, Any]: class TocEvidence: page_index: int source: str - confidence: float reason: str = "" def to_dict(self) -> dict[str, Any]: @@ -133,10 +98,19 @@ class TocResult: toc_pages: list[int] = field(default_factory=list) candidates: list[TocAnchorPage] = field(default_factory=list) evidence: list[TocEvidence] = field(default_factory=list) - method: Literal["toc_marker", "vlm_progressive", "vlm_batch", "visual_scan", "none"] = "none" + method: Literal["vlm_batch", "pdf_outline", "none"] = "none" notes: str = "" failure_kind: TocFailureKind = "none" + @property + def profile_source(self) -> str: + """Honest profile ``source`` label derived from ``method``.""" + if self.method == "pdf_outline": + return "pdf_outline" + if self.method == "vlm_batch": + return "pdf_vlm" + return "none" + def to_dict(self) -> dict[str, Any]: data = asdict(self) data["candidates"] = [candidate.to_dict() for candidate in self.candidates] @@ -144,31 +118,6 @@ def to_dict(self) -> dict[str, Any]: return data -@dataclass -class H1Candidate: - title: str - page: int - confidence: float - matched_line: str - source: Literal["toc_exact_top", "toc_fuzzy_top", "heading_grep", "toc_grep", "h2_refine", "none"] - evidence: dict[str, Any] = field(default_factory=dict) - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -@dataclass -class H1BoundaryResult: - h1_candidates: list[H1Candidate] = field(default_factory=list) - method: Literal["toc_grep", "heading_grep", "none"] = "none" - notes: str = "" - - def to_dict(self) -> dict[str, Any]: - data = asdict(self) - data["h1_candidates"] = [candidate.to_dict() for candidate in self.h1_candidates] - return data - - @dataclass class ValidationReport: valid: bool @@ -191,7 +140,10 @@ class Shard: "toc_leaf_boundary", ] anchor_evidence: str - confidence: float + # Calibrated TOC slice for this shard (heading+level), attached at + # propose.shard_plan. TEXT-TRACK consumes this directly; no printed→physical + # re-filter at parse time. + toc_hierarchies: list[dict[str, Any]] | None = None def to_dict(self) -> dict[str, Any]: return asdict(self) @@ -227,10 +179,8 @@ class PageAnatomyMap: page_labels: list[PageLabel] toc_result: TocResult shard_plan: ShardPlan - h1_result: H1BoundaryResult | None = None document_profile: DocumentProfile | None = None toc_hierarchies: list[dict[str, Any]] | None = None - toc_page_offset: int | None = None skeleton_anchor: dict[str, Any] | None = None skeleton_nodes: list[dict[str, Any]] | None = None pending_skeleton_anchors: list[dict[str, Any]] = field(default_factory=list) @@ -249,12 +199,10 @@ def to_dict(self) -> dict[str, Any]: "page_features": [feature.to_dict() for feature in self.page_features], "page_labels": [label.to_dict() for label in self.page_labels], "toc_result": self.toc_result.to_dict(), - "h1_result": self.h1_result.to_dict() if self.h1_result else None, "shard_plan": self.shard_plan.to_dict(), "document_profile": self.document_profile.to_dict() if self.document_profile else None, - "toc_page_offset": self.toc_page_offset, "skeleton_anchor": self.skeleton_anchor, "skeleton_nodes": self.skeleton_nodes, "pending_skeleton_anchors": list(self.pending_skeleton_anchors), @@ -281,7 +229,6 @@ class ToolContext: pdf_path: str job_id: str blackboard: Any - budget: Any trace: Any output_dir: str | None = None settings: dict[str, Any] = field(default_factory=dict) diff --git a/apps/worker/app/services/document_agent/planner/__init__.py b/apps/worker/app/services/document_agent/planner/__init__.py deleted file mode 100644 index 09de37f57..000000000 --- a/apps/worker/app/services/document_agent/planner/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""One-shot VLM profile planner.""" - -from app.services.document_agent.planner.planner import ( - PAGE_KIND_DEFINITIONS, - ProfilePlanner, - _sample_pages, -) - -__all__ = [ - "PAGE_KIND_DEFINITIONS", - "ProfilePlanner", - "_sample_pages", -] diff --git a/apps/worker/app/services/document_agent/registry.py b/apps/worker/app/services/document_agent/registry.py index 330290d20..0a1387cfa 100644 --- a/apps/worker/app/services/document_agent/registry.py +++ b/apps/worker/app/services/document_agent/registry.py @@ -1,4 +1,4 @@ -"""Agent tool registry with blackboard-based preconditions.""" +"""Profile tool registry with blackboard-based preconditions.""" from __future__ import annotations @@ -6,13 +6,13 @@ from typing import Any, Callable from app.services.document_agent.manifest import ToolContext, ToolResult -from app.services.document_agent.state import AgentBlackboard +from app.services.document_agent.state import ProfileBlackboard ToolHandler = Callable[[ToolContext, dict[str, Any]], ToolResult] -Precondition = Callable[[AgentBlackboard], tuple[bool, str]] +Precondition = Callable[[ProfileBlackboard], tuple[bool, str]] -def _always(_blackboard: AgentBlackboard) -> tuple[bool, str]: +def _always(_blackboard: ProfileBlackboard) -> tuple[bool, str]: return True, "" @@ -24,16 +24,6 @@ class ToolSpec: preconditions: tuple[Precondition, ...] handler: ToolHandler - def to_openai_schema(self) -> dict[str, Any]: - return { - "type": "function", - "function": { - "name": self.name, - "description": self.description, - "parameters": self.parameters, - }, - } - class ToolRegistry: def __init__(self) -> None: @@ -45,14 +35,7 @@ def register(self, spec: ToolSpec) -> None: def get(self, name: str) -> ToolSpec | None: return self._tools.get(name) - def openai_specs(self, blackboard: AgentBlackboard) -> list[dict[str, Any]]: - return [ - tool.to_openai_schema() - for tool in self._tools.values() - if self._preconditions_met(tool, blackboard)[0] - ] - - def allowed_names(self, blackboard: AgentBlackboard) -> list[str]: + def allowed_names(self, blackboard: ProfileBlackboard) -> list[str]: return [ name for name, tool in self._tools.items() @@ -62,7 +45,7 @@ def allowed_names(self, blackboard: AgentBlackboard) -> list[str]: def _preconditions_met( self, tool: ToolSpec, - blackboard: AgentBlackboard, + blackboard: ProfileBlackboard, ) -> tuple[bool, str]: for check in tool.preconditions: ok, reason = check(blackboard) @@ -119,30 +102,26 @@ def _decorator(handler: ToolHandler) -> ToolHandler: return _decorator -def has_page_features(blackboard: AgentBlackboard) -> tuple[bool, str]: +def has_page_features(blackboard: ProfileBlackboard) -> tuple[bool, str]: return bool(blackboard.page_features), "page_features missing; run bootstrap probe first" -def has_page_labels(blackboard: AgentBlackboard) -> tuple[bool, str]: +def has_page_labels(blackboard: ProfileBlackboard) -> tuple[bool, str]: return bool(blackboard.page_labels), "page_labels missing; run bootstrap classify first" -def has_doc_stats(blackboard: AgentBlackboard) -> tuple[bool, str]: +def has_doc_stats(blackboard: ProfileBlackboard) -> tuple[bool, str]: return bool(blackboard.doc_stats), "doc_stats missing; run bootstrap aggregate first" -def has_document_profile(blackboard: AgentBlackboard) -> tuple[bool, str]: - return blackboard.document_profile is not None, "document_profile missing; run planner first" - - -def has_page_full_text(blackboard: AgentBlackboard) -> tuple[bool, str]: +def has_page_full_text(blackboard: ProfileBlackboard) -> tuple[bool, str]: return ( bool(blackboard.page_full_text_cache), "page_full_text_cache missing; run text scan first", ) -def not_is_scanned(blackboard: AgentBlackboard) -> tuple[bool, str]: +def not_is_scanned(blackboard: ProfileBlackboard) -> tuple[bool, str]: profile = blackboard.document_profile return ( profile is not None and not profile.is_scanned, @@ -150,17 +129,9 @@ def not_is_scanned(blackboard: AgentBlackboard) -> tuple[bool, str]: ) -def has_toc_anchors(blackboard: AgentBlackboard) -> tuple[bool, str]: - return bool(blackboard.toc_anchor_pages), "toc anchors missing; call find_toc_anchors first" - - -def has_toc_result(blackboard: AgentBlackboard) -> tuple[bool, str]: +def has_toc_result(blackboard: ProfileBlackboard) -> tuple[bool, str]: return blackboard.toc_result is not None, "toc_result missing; call extract_toc first" -def has_toc_hierarchies(blackboard: AgentBlackboard) -> tuple[bool, str]: - return bool(blackboard.toc_hierarchies), "toc_hierarchies missing; call extract_toc first" - - -def has_shard_plan(blackboard: AgentBlackboard) -> tuple[bool, str]: +def has_shard_plan(blackboard: ProfileBlackboard) -> tuple[bool, str]: return blackboard.shard_plan is not None, "shard_plan missing; call propose_shard first" diff --git a/apps/worker/app/services/document_agent/state.py b/apps/worker/app/services/document_agent/state.py index 93ddfd83d..67b434022 100644 --- a/apps/worker/app/services/document_agent/state.py +++ b/apps/worker/app/services/document_agent/state.py @@ -1,4 +1,4 @@ -"""State carried by the document profile agent.""" +"""State carried by the document profile workflow.""" from __future__ import annotations @@ -7,18 +7,17 @@ from typing import Any from app.services.document_agent.manifest import ( - AgentVerdict, DocumentProfile, - H1BoundaryResult, PageFeature, PageLabel, + ProfileVerdict, ShardPlan, TocAnchorPage, TocResult, ) -class DocumentAgentState(str, Enum): +class ProfileState(str, Enum): INIT = "init" RUNNING = "running" READY = "ready" @@ -26,7 +25,7 @@ class DocumentAgentState(str, Enum): @dataclass -class AgentBlackboard: +class ProfileBlackboard: page_count: int = 0 document_profile: DocumentProfile | None = None page_features: list[PageFeature] = field(default_factory=list) @@ -34,18 +33,14 @@ class AgentBlackboard: doc_stats: dict[str, Any] = field(default_factory=dict) extrema_pages: list[int] = field(default_factory=list) toc_anchor_pages: list[TocAnchorPage] = field(default_factory=list) + pdf_outline_roots: list[dict[str, Any]] | None = None toc_result: TocResult | None = None toc_hierarchies: list[dict[str, Any]] | None = None - h1_result: H1BoundaryResult | None = None - toc_page_offset: int | None = None skeleton_anchor: dict[str, Any] | None = None skeleton_nodes: list[dict[str, Any]] | None = None pending_skeleton_anchors: list[dict[str, Any]] = field(default_factory=list) shard_plan: ShardPlan | None = None validation_report: dict[str, Any] | None = None - verdict: AgentVerdict | None = None - step_history: list[dict[str, Any]] = field(default_factory=list) + verdict: ProfileVerdict | None = None page_full_text_cache: dict[int, str] = field(default_factory=dict) global_signals: dict[str, Any] = field(default_factory=dict) - errors: list[str] = field(default_factory=list) - diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index d0865bde2..91a11b7b4 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -1,5 +1,5 @@ """Shared hierarchy anchoring: Phase-2 bulk/bisect/null-page + SkeletonAnchor. -Phase-1 offset discovery lives in ``document_agent.agents.calibration``. +Phase-1 offset discovery lives in ``document_agent.calibration``. ``anchor_hierarchy`` composes Phase-1 + Phase-2 for production callers. """ @@ -17,12 +17,23 @@ last_leaf_start_under, locate_title_compact_strict, ) -from app.services.document_agent.structure.page_locate_agent import ( +from app.services.document_agent.structure.section_page_verify import ( verify_section_page_choice, ) from loguru import logger +def _first_sibling_null_parent_scan_start(right: int) -> int: + """Left edge for first-at-level null parents: at most one 2+4+6+10 budget. + + Does not inherit a wider parent/body scope. Floors at document page 1. + """ + from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE + + budget = sum(DEFAULT_WINDOW_SCHEDULE) + return max(1, int(right) - budget + 1) + + def prune_out_of_scope_nodes( nodes: list[TitleNode], *, @@ -144,9 +155,7 @@ def toc_range_end(hierarchy: dict[str, Any]) -> int | None: return None -# ── Null-page parent locate (compact-strict + RTL visual) ─────────────────── - -_NULL_PARENT_VISUAL_CONFIDENCE = 0.6 +# ── Null-page parent locate (sibling window / first-sibling scan) ─────────── def locate_null_page_parent_overrides( @@ -159,9 +168,13 @@ def locate_null_page_parent_overrides( ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: """Locate TOC parents with ``printed_page=None`` into ``match_overrides``. - Window for parent P: ``[last leaf start under previous same-level sibling, - first leaf start under P]``. Text path is compact→strict unique page; on - miss/ambiguity, scan right→left with ``verify_section_page_choice``. + Window for parent P with a previous same-level sibling: ``[last leaf under + that sibling, first leaf under P]``; text then RTL visual verify. + + First-at-level parents (no left sibling) do **not** inherit a wider parent + or body scope. Left edge is one Phase-1 ``2+4+6+10`` budget before the first + child (floor page 1). Text runs in that window; on miss, reuse + ``scan_title_forward`` (same schedule, early exit). Miss → unresolved. Returns ``(overrides, report)`` where *report* lists every null-page parent attempt (for debug / LLM-call accounting). @@ -186,14 +199,6 @@ def walk( and node.printed_page is None and path_titles not in out ): - if index > 0: - left = last_leaf_start_under( - sibling_nodes[index - 1], parent_titles, out - ) - if left is None: - left = scope_start - else: - left = scope_start right = first_leaf_start_under(node, parent_titles, out) entry: dict[str, Any] = { "path_titles": list(path_titles), @@ -205,58 +210,56 @@ def walk( "accept": None, "visual_verify_calls": 0, } - if right is None or right < left: + if right is None: report.append(entry) logger.info( "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child left={}", + "title={!r} reason=no_located_first_child", node.title, - left, ) - else: - entry["window"] = [left, right] - scope_pages = [ - page for page in body_pages if left <= page <= right - ] - match = locate_title_compact_strict( - node.title, - scope_pages=scope_pages, - page_texts=page_texts, + elif index > 0: + left = last_leaf_start_under( + sibling_nodes[index - 1], parent_titles, out ) - visual_calls = 0 - if match is None and ctx is not None: - match, visual_calls = _visual_rtl_locate_parent( + if left is None: + left = scope_start + if right < left: + report.append(entry) + logger.info( + "[structure_anchoring] null-page parent skipped: " + "title={!r} reason=no_located_first_child left={}", + node.title, + left, + ) + else: + _resolve_null_parent_with_sibling_window( + path_titles=path_titles, title=node.title, left=left, right=right, + body_pages=body_pages, body_set=body_set, + page_texts=page_texts, ctx=ctx, + out=out, + entry=entry, + report=report, ) - entry["visual_verify_calls"] = visual_calls - if match is not None and match.page in body_set: - out[path_titles] = match - entry["result"] = str(match.evidence.get("accept") or match.source) - entry["page"] = match.page - entry["accept"] = match.evidence.get("accept") - logger.info( - "[structure_anchoring] null-page parent located: " - "title={!r} page={} window={} accept={} visual_calls={}", - node.title, - match.page, - [left, right], - match.evidence.get("accept"), - visual_calls, - ) - else: - entry["result"] = "unresolved" - logger.info( - "[structure_anchoring] null-page parent unresolved: " - "title={!r} window={} visual_calls={}", - node.title, - [left, right], - visual_calls, - ) - report.append(entry) + else: + left = _first_sibling_null_parent_scan_start(right) + _resolve_null_parent_first_sibling( + path_titles=path_titles, + title=node.title, + left=left, + right=right, + body_pages=body_pages, + body_set=body_set, + page_texts=page_texts, + ctx=ctx, + out=out, + entry=entry, + report=report, + ) if node.children: child_scope_start = ( out[path_titles].page if path_titles in out else scope_start @@ -275,6 +278,152 @@ def walk( return out, report +def _record_null_parent_outcome( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + match: TitleMatch | None, + visual_calls: int, + body_set: set[int], + out: dict[tuple[str, ...], TitleMatch], + entry: dict[str, Any], + report: list[dict[str, Any]], +) -> None: + entry["window"] = [left, right] + entry["visual_verify_calls"] = visual_calls + if match is not None and match.page in body_set: + out[path_titles] = match + entry["result"] = str(match.evidence.get("accept") or match.source) + entry["page"] = match.page + entry["accept"] = match.evidence.get("accept") + logger.info( + "[structure_anchoring] null-page parent located: " + "title={!r} page={} window={} accept={} visual_calls={}", + title, + match.page, + [left, right], + match.evidence.get("accept"), + visual_calls, + ) + else: + entry["result"] = "unresolved" + logger.info( + "[structure_anchoring] null-page parent unresolved: " + "title={!r} window={} visual_calls={}", + title, + [left, right], + visual_calls, + ) + report.append(entry) + + +def _resolve_null_parent_with_sibling_window( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + body_pages: list[int], + body_set: set[int], + page_texts: dict[int, str], + ctx: ToolContext | None, + out: dict[tuple[str, ...], TitleMatch], + entry: dict[str, Any], + report: list[dict[str, Any]], +) -> None: + scope_pages = [page for page in body_pages if left <= page <= right] + match = locate_title_compact_strict( + title, + scope_pages=scope_pages, + page_texts=page_texts, + ) + visual_calls = 0 + if match is None and ctx is not None: + match, visual_calls = _visual_rtl_locate_parent( + title=title, + left=left, + right=right, + body_set=body_set, + ctx=ctx, + ) + _record_null_parent_outcome( + path_titles=path_titles, + title=title, + left=left, + right=right, + match=match, + visual_calls=visual_calls, + body_set=body_set, + out=out, + entry=entry, + report=report, + ) + + +def _resolve_null_parent_first_sibling( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + body_pages: list[int], + body_set: set[int], + page_texts: dict[int, str], + ctx: ToolContext | None, + out: dict[tuple[str, ...], TitleMatch], + entry: dict[str, Any], + report: list[dict[str, Any]], +) -> None: + """First-at-level null parent: capped text window, then ``scan_title_forward``.""" + from app.services.document_agent.calibration.scan import ( + DEFAULT_WINDOW_SCHEDULE, + scan_title_forward, + ) + + scope_pages = [page for page in body_pages if left <= page <= right] + match = locate_title_compact_strict( + title, + scope_pages=scope_pages, + page_texts=page_texts, + ) + visual_calls = 0 + if match is None and ctx is not None: + scan = scan_title_forward( + ctx=ctx, + title=title, + start_page=left, + page_count=right, + window_schedule=DEFAULT_WINDOW_SCHEDULE, + ) + visual_calls = len(scan.scanned_pages) + if scan.found and scan.found_page is not None: + match = TitleMatch( + page=int(scan.found_page), + source="inspect_vlm", + matched_line="", + candidates=[int(scan.found_page)], + evidence={ + "accept": "scan_forward", + "null_page_parent_probe": True, + "scanned_pages": list(scan.scanned_pages), + }, + ) + _record_null_parent_outcome( + path_titles=path_titles, + title=title, + left=left, + right=right, + match=match, + visual_calls=visual_calls, + body_set=body_set, + out=out, + entry=entry, + report=report, + ) + + def _visual_rtl_locate_parent( *, title: str, @@ -290,10 +439,8 @@ def _visual_rtl_locate_parent( continue candidate = TitleMatch( page=page, - confidence=0.4, - source="agent_heuristic", + source="inspect_vlm", matched_line="", - score=0.4, candidates=[page], evidence={"null_page_parent_probe": True}, ) @@ -305,33 +452,13 @@ def _visual_rtl_locate_parent( candidate_page_cap=1, ) selected = result.get("selected_page") - confidence = float(result.get("confidence") or 0.0) - if selected != page or confidence < _NULL_PARENT_VISUAL_CONFIDENCE: + if selected != page: continue - if result.get("source") == "agent_vlm": - return ( - TitleMatch( - page=page, - confidence=confidence, - source="agent_vlm", - matched_line="", - score=confidence, - candidates=[page], - evidence={ - "accept": "visual_rtl", - "reason": result.get("reason", ""), - "visual_verify_calls": visual_calls, - }, - ), - visual_calls, - ) return ( TitleMatch( page=page, - confidence=confidence, - source="agent_heuristic", + source="inspect_vlm", matched_line="", - score=confidence, candidates=[page], evidence={ "accept": "visual_rtl", @@ -346,9 +473,7 @@ def _visual_rtl_locate_parent( # ── Offset-guided bulk anchoring with recursive recalibrate (Phase-2) ─────── -_TAIL_VERIFY_CONFIDENCE_THRESHOLD = 0.6 _MAX_RECALIBRATE_DEPTH = 5 -_MAX_RECALIBRATE_DELTA = 5 def _verify_offset_tail( @@ -396,10 +521,8 @@ def _verify_offset_tail( candidate = TitleMatch( page=expected_page, - confidence=0.4, - source="agent_heuristic", + source="inspect_vlm", matched_line="", - score=0.4, candidates=[expected_page], evidence={"tail_verify_probe": True}, ) @@ -409,16 +532,12 @@ def _verify_offset_tail( candidate_matches=[candidate], candidate_page_cap=1, ) - confirmed = ( - result.get("selected_page") == expected_page - and result.get("confidence", 0) >= _TAIL_VERIFY_CONFIDENCE_THRESHOLD - ) + confirmed = result.get("selected_page") == expected_page logger.info( - "[structure_anchoring] tail verify: title={!r} expected_page={} confirmed={} confidence={}", + "[structure_anchoring] tail verify: title={!r} expected_page={} confirmed={}", node.title, expected_page, confirmed, - result.get("confidence", 0), ) return confirmed @@ -435,10 +554,8 @@ def _vlm_confirm_single_page( return False candidate = TitleMatch( page=expected_page, - confidence=0.4, - source="agent_heuristic", + source="inspect_vlm", matched_line="", - score=0.4, candidates=[expected_page], evidence={"bisect_probe": True}, ) @@ -448,10 +565,8 @@ def _vlm_confirm_single_page( candidate_matches=[candidate], candidate_page_cap=1, ) - return ( - result.get("selected_page") == expected_page - and result.get("confidence", 0) >= _TAIL_VERIFY_CONFIDENCE_THRESHOLD - ) + return result.get("selected_page") == expected_page + def _bisect_offset_breakpoint( @@ -461,10 +576,17 @@ def _bisect_offset_breakpoint( ctx: ToolContext, page_count: int, ) -> int: - """Binary search for the last leaf index where offset is valid. O(log n) VLM calls.""" + """Return the last leaf index where ``offset`` holds, or ``-1`` if none do. + + Does not assume the first leaf is valid: every candidate index is confirmed + (or rejected) before it can become the breakpoint. O(log n) VLM calls. + """ + if not leaves: + return -1 lo, hi = 0, len(leaves) - 1 - while lo < hi: - mid = (lo + hi + 1) // 2 + last_valid = -1 + while lo <= hi: + mid = (lo + hi) // 2 _, node = leaves[mid] if node.printed_page is None: hi = mid - 1 @@ -473,15 +595,16 @@ def _bisect_offset_breakpoint( if _vlm_confirm_single_page( ctx=ctx, title=node.title, expected_page=expected, page_count=page_count ): - lo = mid + last_valid = mid + lo = mid + 1 else: hi = mid - 1 logger.info( "[structure_anchoring] bisect breakpoint: last_valid_index={} / total={}", - lo, + last_valid, len(leaves), ) - return lo + return last_valid def bulk_offset_matches( @@ -496,13 +619,10 @@ def bulk_offset_matches( page = node.printed_page + offset matches[path_titles] = TitleMatch( page=page, - confidence=0.88, - source="agent_vlm", + source="bulk_offset", matched_line="", - score=0.88, candidates=[page], evidence={ - "bulk_offset": True, "offset": offset, "printed_page": node.printed_page, }, @@ -510,6 +630,76 @@ def bulk_offset_matches( return matches +def _iter_printed_page_parents( + nodes: list[TitleNode], + *, + parent_titles: tuple[str, ...] = (), +) -> list[tuple[tuple[str, ...], TitleNode]]: + """DFS non-leaf nodes that print their own page in the TOC.""" + parents: list[tuple[tuple[str, ...], TitleNode]] = [] + for node in nodes: + path_titles = (*parent_titles, node.title) + if not node.children: + continue + if node.printed_page is not None: + parents.append((path_titles, node)) + parents.extend( + _iter_printed_page_parents(node.children, parent_titles=path_titles) + ) + return parents + + +def _descendant_regime_offset( + node: TitleNode, + path_titles: tuple[str, ...], + matches: dict[tuple[str, ...], TitleMatch], +) -> int | None: + """Offset of the parent's first anchored descendant leaf, i.e. its regime.""" + for leaf_path, _leaf in iter_leaf_title_nodes( + node.children, parent_titles=path_titles + ): + match = matches.get(leaf_path) + if match is None: + continue + offset = match.evidence.get("offset") + if offset is not None: + return int(offset) + return None + + +def backfill_parent_offset_matches( + *, + nodes: list[TitleNode], + matches: dict[tuple[str, ...], TitleMatch], + page_count: int, +) -> dict[tuple[str, ...], TitleMatch]: + """Anchor TOC parents that print a page, reusing their descendant's offset. + + Bulk anchoring consumes leaves only, so a TOC whose section headings carry + printed pages leaves every parent without a physical page. The parent shares + the calibration regime of its first anchored descendant, so ``printed_page + + that regime's offset`` is the parent's physical page. + """ + by_offset: dict[int, list[tuple[tuple[str, ...], TitleNode]]] = {} + for path_titles, node in _iter_printed_page_parents(nodes): + if path_titles in matches: + continue + offset = _descendant_regime_offset(node, path_titles, matches) + if offset is None: + continue + by_offset.setdefault(offset, []).append((path_titles, node)) + + out: dict[tuple[str, ...], TitleMatch] = {} + for offset, group in by_offset.items(): + for path_titles, match in bulk_offset_matches(group, offset).items(): + if 1 <= match.page <= page_count: + out[path_titles] = replace( + match, + evidence={**match.evidence, "parent_backfill": True}, + ) + return out + + def _recalibrate_after_breakpoint( *, entry_node: TitleNode, @@ -517,29 +707,59 @@ def _recalibrate_after_breakpoint( ctx: ToolContext, page_count: int, ) -> int | None: - """Probe offsets old_offset+1, +2, ... to find new offset after breakpoint. + """Re-find offset for the first remaining leaf after a breakpoint. - Monotonicity guarantees new offset > old offset, so search space is tiny. + Same mechanic as Phase-1: forward ``inspect.pages`` scan until the title + START is found. Monotonicity says the physical page is strictly after the + failed ``printed + old_offset`` slot, so the scan cursor starts there. """ entry_printed_page = entry_node.printed_page if entry_printed_page is None: return None - for delta in range(1, _MAX_RECALIBRATE_DELTA + 1): - new_offset = old_offset + delta - if _vlm_confirm_single_page( - ctx=ctx, - title=entry_node.title, - expected_page=entry_printed_page + new_offset, - page_count=page_count, - ): - logger.info( - "[structure_anchoring] recalibrate: title={!r} new_offset={} (delta=+{})", - entry_node.title, - new_offset, - delta, - ) - return new_offset - return None + + # Lazy import: scan → inspect_pages → tools must not load at module import. + from app.services.document_agent.calibration.scan import scan_title_forward + + start_page = entry_printed_page + old_offset + 1 + if start_page > page_count: + return None + + scan = scan_title_forward( + ctx=ctx, + title=entry_node.title, + start_page=start_page, + page_count=page_count, + ) + if not scan.found or scan.found_page is None: + logger.info( + "[structure_anchoring] recalibrate miss: title={!r} start={} scanned={}", + entry_node.title, + start_page, + scan.scanned_pages, + ) + return None + + new_offset = int(scan.found_page) - entry_printed_page + if new_offset <= old_offset: + logger.info( + "[structure_anchoring] recalibrate rejected non-monotonic offset: " + "title={!r} old={} new={} found_page={}", + entry_node.title, + old_offset, + new_offset, + scan.found_page, + ) + return None + + logger.info( + "[structure_anchoring] recalibrate: title={!r} new_offset={} " + "(delta=+{}, found_page={})", + entry_node.title, + new_offset, + new_offset - old_offset, + scan.found_page, + ) + return new_offset def offset_guided_anchoring( @@ -555,13 +775,14 @@ def offset_guided_anchoring( Strategy: 1. Tail verify last leaf with current offset 2. If pass → bulk apply all leaves (Theorem 1) - 3. If fail → binary search for breakpoint - 4. Bulk apply leaves before breakpoint - 5. Recalibrate: probe remaining[0] with offset+1, +2, ... (monotonicity) + 3. If fail → binary search for last valid index (``-1`` if none) + 4. Bulk apply only verified prefix (empty when breakpoint is ``-1``) + 5. Recalibrate remaining[0] via Phase-1 forward scan from printed+old+1 6. Recurse on remaining segment with new offset - 7. If recalibrate fails → return partial (caller falls back for remainder) + 7. If recalibrate fails → keep prefix only (caller prunes the rest) - Returns match_overrides for all anchored leaves, or None for full fallback. + Returns match_overrides for anchored leaves (including Phase-1 seeds), or + None when nothing was anchored. """ leaves = [ (path, node) @@ -617,13 +838,15 @@ def _anchor_segment_recursive( matches.update(bulk) return - bp = _bisect_offset_breakpoint(leaves=leaves, offset=offset, ctx=ctx, page_count=page_count) - confirmed_leaves = leaves[: bp + 1] + bp = _bisect_offset_breakpoint( + leaves=leaves, offset=offset, ctx=ctx, page_count=page_count + ) + # bp == -1 → no leaf confirmed under this offset; do not invent a prefix. + confirmed_leaves = leaves[: bp + 1] if bp >= 0 else [] if confirmed_leaves: - bulk = bulk_offset_matches(confirmed_leaves, offset) - matches.update(bulk) + matches.update(bulk_offset_matches(confirmed_leaves, offset)) - remaining = leaves[bp + 1:] + remaining = leaves[bp + 1 :] if bp >= 0 else list(leaves) if not remaining: return @@ -655,16 +878,15 @@ class SkeletonAnchor: null_page_report: list[dict[str, Any]] bulk_count: int pruned_count: int = 0 - locate_agent: str = "offset_only" + locate_method: str = "offset_only" + source: str = "" def serialize_title_match(match: TitleMatch) -> dict[str, Any]: return { "page": match.page, - "confidence": match.confidence, "source": match.source, "matched_line": match.matched_line, - "score": match.score, "candidates": list(match.candidates), "evidence": dict(match.evidence or {}), } @@ -683,17 +905,16 @@ def serialize_skeleton_anchor(anchor: SkeletonAnchor) -> dict[str, Any]: "null_page_report": list(anchor.null_page_report or []), "bulk_count": int(anchor.bulk_count or 0), "pruned_count": int(anchor.pruned_count or 0), - "locate_agent": anchor.locate_agent, + "locate_method": anchor.locate_method, + "source": anchor.source, } def deserialize_title_match(data: dict[str, Any]) -> TitleMatch: return TitleMatch( page=int(data["page"]), - confidence=float(data.get("confidence") or 0.0), - source=data.get("source") or "agent_vlm", # type: ignore[arg-type] + source=str(data.get("source") or "bulk_offset"), # type: ignore[arg-type] matched_line=str(data.get("matched_line") or ""), - score=float(data.get("score") or 0.0), candidates=[int(p) for p in (data.get("candidates") or [])], evidence=dict(data.get("evidence") or {}), ) @@ -706,7 +927,6 @@ def serialize_title_node(node: TitleNode) -> dict[str, Any]: "printed_page": node.printed_page, "printed_label": node.printed_label, "page_kind": node.page_kind, - "physical_page_hint": node.physical_page_hint, "children": [serialize_title_node(child) for child in node.children], } @@ -719,7 +939,6 @@ def deserialize_title_node(data: dict[str, Any]) -> TitleNode: if isinstance(child, dict) ] printed_page = data.get("printed_page") - physical_page_hint = data.get("physical_page_hint") return TitleNode( title=str(data.get("title") or ""), level=int(data.get("level") or 1), @@ -730,9 +949,6 @@ def deserialize_title_node(data: dict[str, Any]) -> TitleNode: page_kind=data.get("page_kind") if isinstance(data.get("page_kind"), str) else None, - physical_page_hint=( - None if physical_page_hint is None else int(physical_page_hint) - ), children=children, ) @@ -759,7 +975,8 @@ def deserialize_skeleton_anchor(data: dict[str, Any]) -> SkeletonAnchor: null_page_report=list(data.get("null_page_report") or []), bulk_count=int(data.get("bulk_count") or 0), pruned_count=int(data.get("pruned_count") or 0), - locate_agent=str(data.get("locate_agent") or "offset_only"), + locate_method=str(data.get("locate_method") or "offset_only"), + source=str(data.get("source") or ""), ) @@ -775,7 +992,7 @@ def anchor_hierarchy_from_offset( ) -> tuple[list[TitleNode], SkeletonAnchor]: """Production prune → bulk → null-page given a precomputed offset. - Phase-2 entry after Agent ``calibrate_offset`` (Phase-1). + Phase-2 entry after ``calibrate_offset`` (Phase-1). """ seed_overrides = dict(calibration_overrides or {}) pruned_count = 0 @@ -797,11 +1014,11 @@ def anchor_hierarchy_from_offset( if offset_matches is not None: match_overrides = offset_matches - locate_agent = "offset_guided_bulk" + locate_method = "offset_guided_bulk" bulk_count = len(offset_matches) else: match_overrides = seed_overrides - locate_agent = "offset_only" + locate_method = "offset_only" bulk_count = 0 working, unanchored_removed = prune_unanchored_toc_leaves( @@ -829,5 +1046,5 @@ def anchor_hierarchy_from_offset( null_page_report=null_page_report, bulk_count=bulk_count, pruned_count=pruned_count, - locate_agent=locate_agent, + locate_method=locate_method, ) diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index 86719c1d1..bf94692b8 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -1,9 +1,10 @@ """Locate hierarchy titles on PDF pages and resolve page ranges. -Deterministic title anchoring and range assembly. Leaf starts come from -offset-guided ``match_overrides`` or per-line strict exact. Null-page parents -are located upstream via compact-strict (cross-line) + optional VLM, then -resolved here including parent self-only spans for interstitial pages. +Deterministic range assembly from PROFILE ``match_overrides``. Leaf starts +come only from those overrides. Null-page parents are located upstream via +compact-strict (cross-line) + optional VLM, then resolved here including +parent self-only spans for interstitial pages. Parents without an override +may still inherit start from the earliest located descendant leaf. """ from __future__ import annotations @@ -19,9 +20,10 @@ TitleMatchSource = Literal[ "anchored", - "h1_result", - "agent_vlm", - "agent_heuristic", + "bulk_offset", + "inspect_vlm", + "inferred_descendant", + "pdf_outline", ] @@ -108,17 +110,14 @@ class TitleNode: printed_page: int | None = None printed_label: str | None = None page_kind: str | None = None - physical_page_hint: int | None = None children: list["TitleNode"] = field(default_factory=list) @dataclass(frozen=True) class TitleMatch: page: int - confidence: float source: TitleMatchSource matched_line: str - score: float candidates: list[int] evidence: dict[str, Any] = field(default_factory=dict) @@ -134,33 +133,6 @@ class ResolvedHierarchyRange: evidence: dict[str, Any] = field(default_factory=dict) -@dataclass(frozen=True) -class _LineHit: - page: int - line_index: int - line: str - source: TitleMatchSource - score: float - - -def locate_title_strict_exact( - title: str, - *, - scope_pages: list[int], - page_texts: dict[int, str], -) -> TitleMatch | None: - """Return a direct anchor only when a cleaned heading line has one page hit.""" - hits = _find_anchored_hits(title, scope_pages, page_texts) - pages = sorted({hit.page for hit in hits}) - if len(pages) != 1: - return None - return _choose_best_hit( - hits, - source="anchored", - extra_evidence={"accept": "strict_exact_unique"}, - ) - - def locate_title_compact_strict( title: str, *, @@ -194,10 +166,8 @@ def locate_title_compact_strict( page = unique_pages[0] return TitleMatch( page=page, - confidence=0.92, source="anchored", matched_line=matched_preview, - score=0.96, candidates=[page], evidence={"accept": "compact_strict_unique"}, ) @@ -239,7 +209,6 @@ def resolve_hierarchy_page_ranges( nodes: list[TitleNode], *, page_count: int, - page_texts: dict[int, str], body_pages: list[int] | None = None, match_overrides: dict[tuple[str, ...], TitleMatch] | None = None, ) -> list[ResolvedHierarchyRange]: @@ -247,6 +216,7 @@ def resolve_hierarchy_page_ranges( Emits leaf ranges and parent self-only spans when a parent start is strictly before its first located descendant leaf. Ranges are closed-closed. + Leaf starts come only from ``match_overrides`` (PROFILE anchoring). """ if page_count <= 0 or not nodes: return [] @@ -264,13 +234,50 @@ def resolve_hierarchy_page_ranges( parent_scope=scope, allowed_pages=allowed_pages, parent_titles=(), - page_texts=page_texts, match_overrides=match_overrides or {}, resolved=resolved, ) return resolved +def coverage_by_path( + ranges: list[ResolvedHierarchyRange], +) -> dict[tuple[str, ...], tuple[int, int]]: + """Aggregate resolved ranges into one closed span per ancestor path. + + ``resolve_hierarchy_page_ranges`` emits leaves (plus parent self-only + spans), so an ancestor's span is the union of its descendants' ranges. + """ + coverage: dict[tuple[str, ...], tuple[int, int]] = {} + for item in ranges: + for depth in range(1, len(item.path_titles) + 1): + path = item.path_titles[:depth] + span = coverage.get(path) + if span is None: + coverage[path] = (item.start_page, item.end_page) + else: + coverage[path] = ( + min(span[0], item.start_page), + max(span[1], item.end_page), + ) + return coverage + + +def deepest_covering_path( + coverage: dict[tuple[str, ...], tuple[int, int]], + *, + start: int, + end: int, +) -> tuple[str, ...] | None: + """Deepest path whose span contains the closed window ``[start, end]``.""" + found: tuple[str, ...] | None = None + for path, span in coverage.items(): + if span[0] <= start and end <= span[1]: + if found is None or len(path) > len(found): + found = path + return found + + def extract_toc_nodes(toc_hierarchies: list[dict[str, Any]] | None) -> list[TitleNode]: """Build a title tree from supported TOC hierarchy payloads.""" flat_entries: list[dict[str, Any]] = [] @@ -288,7 +295,6 @@ def _resolve_siblings( parent_scope: PageRange, allowed_pages: set[int], parent_titles: tuple[str, ...], - page_texts: dict[int, str], match_overrides: dict[tuple[str, ...], TitleMatch], resolved: list[ResolvedHierarchyRange], ) -> None: @@ -302,7 +308,6 @@ def _resolve_siblings( node, path_titles=path_titles, scope_pages=pages, - page_texts=page_texts, match_overrides=match_overrides, ) if match is None: @@ -319,7 +324,6 @@ def _resolve_siblings( lower_bound=lower_bound, parent_end=parent_scope.end, allowed_pages=allowed_pages, - page_texts=page_texts, match_overrides=match_overrides, parent_titles=parent_titles, ) @@ -370,7 +374,6 @@ def _resolve_siblings( parent_scope=PageRange(start_page, end_page), allowed_pages=allowed_pages, parent_titles=path_titles, - page_texts=page_texts, match_overrides=match_overrides, resolved=resolved, ) @@ -394,27 +397,18 @@ def _locate_match_for_node( *, path_titles: tuple[str, ...], scope_pages: list[int], - page_texts: dict[int, str], match_overrides: dict[tuple[str, ...], TitleMatch], ) -> TitleMatch | None: match = _match_override(path_titles, match_overrides, scope_pages) - if match is not None: - return match - match = _match_physical_hint(node=node, scope_pages=scope_pages) if match is not None: return match if node.children: - # Parent active locate is upstream (compact-strict / visual). Wide-window - # strict_exact is intentionally not used here. + # Parent active locate is upstream (compact-strict / visual). return _infer_start_from_descendant_overrides( node, parent_titles=path_titles[:-1], match_overrides=match_overrides, scope_pages=scope_pages, ) - return locate_title_strict_exact( - node.title, - scope_pages=scope_pages, - page_texts=page_texts, - ) + return None def _find_next_located_sibling( @@ -424,7 +418,6 @@ def _find_next_located_sibling( lower_bound: int, parent_end: int, allowed_pages: set[int], - page_texts: dict[int, str], match_overrides: dict[tuple[str, ...], TitleMatch], parent_titles: tuple[str, ...], ) -> TitleMatch | None: @@ -435,7 +428,6 @@ def _find_next_located_sibling( sibling, path_titles=path_titles, scope_pages=pages, - page_texts=page_texts, match_overrides=match_overrides, ) if match is not None: @@ -468,14 +460,12 @@ def _infer_start_from_descendant_overrides( return None return TitleMatch( page=min_match.page, - confidence=min(min_match.confidence, 0.80), - source=min_match.source, + source="inferred_descendant", matched_line="", - score=min(min_match.score, 0.80), candidates=[min_match.page], evidence={ "inferred_from": "descendant_leaf_override", - "original_confidence": min_match.confidence, + "leaf_source": min_match.source, "status": "degraded", }, ) @@ -519,13 +509,11 @@ def _next_located_start( def _range_evidence(match: TitleMatch | None) -> dict[str, Any]: if match is None: - return {"source": "unlocated", "confidence": 0.0, "candidates": []} + return {"source": "unlocated", "candidates": []} return { "source": match.source, - "confidence": match.confidence, "matched_line": match.matched_line, "candidates": match.candidates, - "score": match.score, **match.evidence, } @@ -556,24 +544,6 @@ def _unlocated_warning_evidence( } -def _match_physical_hint( - *, - node: TitleNode, - scope_pages: list[int], -) -> TitleMatch | None: - if node.physical_page_hint is None or node.physical_page_hint not in scope_pages: - return None - return TitleMatch( - page=node.physical_page_hint, - confidence=0.88, - source="h1_result", - matched_line="", - score=0.88, - candidates=[node.physical_page_hint], - evidence={"physical_page_hint": node.physical_page_hint}, - ) - - def _allowed_pages_between(start: int, end: int, allowed_pages: set[int]) -> list[int]: if end < start: return [] @@ -584,83 +554,6 @@ def _compact_match_text(text: str) -> str: return re.sub(r"\s+", "", normalize_heading_text(text)).casefold() -def _find_anchored_hits( - title: str, - scope_pages: list[int], - page_texts: dict[int, str], -) -> list[_LineHit]: - hits: list[_LineHit] = [] - needle = normalize_heading_text(clean_toc_title(title) or title).casefold() - if not needle: - return hits - for page, line_index, line in _iter_lines(scope_pages, page_texts): - cleaned_line = normalize_heading_text(clean_toc_title(line)).casefold() - if cleaned_line == needle: - hits.append( - _LineHit( - page=page, - line_index=line_index, - line=line.strip(), - source="anchored", - score=_line_score(line=line, line_index=line_index, base=0.96), - ) - ) - return hits - - -def _choose_best_hit( - hits: list[_LineHit], - *, - source: TitleMatchSource, - extra_evidence: dict[str, Any] | None = None, -) -> TitleMatch: - ordered = sorted( - hits, - key=lambda hit: (hit.score, -hit.line_index, -hit.page), - reverse=True, - ) - best = ordered[0] - pages = sorted({hit.page for hit in ordered}) - confidence_by_source = { - "anchored": 0.92, - "h1_result": 0.88, - "agent_vlm": 0.75, - "agent_heuristic": 0.5, - } - return TitleMatch( - page=best.page, - confidence=confidence_by_source[source], - source=source, - matched_line=best.line[:160], - score=best.score, - candidates=pages, - evidence={ - "line_index": best.line_index, - "candidate_count": len(pages), - **(extra_evidence or {}), - }, - ) - - -def _line_score(*, line: str, line_index: int, base: float) -> float: - stripped = normalize_heading_text(line) - short_line_bonus = max(0.0, 1.0 - (len(stripped) / 140.0)) - top_bonus = max(0.0, 1.0 - (line_index / 18.0)) - return base + short_line_bonus * 0.12 + top_bonus * 0.1 - - -def _iter_lines( - scope_pages: list[int], - page_texts: dict[int, str], -) -> list[tuple[int, int, str]]: - rows: list[tuple[int, int, str]] = [] - for page in scope_pages: - for line_index, line in enumerate(page_texts.get(page, "").splitlines()): - if line.strip(): - rows.append((page, line_index, line)) - return rows - - def _extract_flat_entries(payload: Any) -> list[dict[str, Any]]: if isinstance(payload, list): return [ @@ -788,9 +681,6 @@ def _collapse(node: TitleNode) -> TitleNode: merged_printed_page = only_child.printed_page or node.printed_page merged_printed_label = only_child.printed_label or node.printed_label merged_page_kind = only_child.page_kind or node.page_kind - merged_physical_hint = ( - only_child.physical_page_hint or node.physical_page_hint - ) promoted = [ _replace(gc, level=max(1, gc.level - 1)) for gc in only_child.children @@ -801,7 +691,6 @@ def _collapse(node: TitleNode) -> TitleNode: printed_page=merged_printed_page, printed_label=merged_printed_label, page_kind=merged_page_kind, - physical_page_hint=merged_physical_hint, children=promoted, ) diff --git a/apps/worker/app/services/document_agent/structure/outline_check.py b/apps/worker/app/services/document_agent/structure/outline_check.py new file mode 100644 index 000000000..d3975af8e --- /dev/null +++ b/apps/worker/app/services/document_agent/structure/outline_check.py @@ -0,0 +1,105 @@ +"""PDF outline self-check and compact tree digest for VLM confirm.""" + +from __future__ import annotations + +import re +from typing import Any + +from app.services.document_parser.structure.body_boundary import ( + clean_toc_title, + normalize_heading_text, +) + +_DEFAULT_DIGEST_TITLE_CHARS = 80 + + +def flatten_outline_entries(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Flatten nested outline roots into ``heading`` / ``level`` / ``page`` rows.""" + entries: list[dict[str, Any]] = [] + + def walk(items: list[dict[str, Any]]) -> None: + for node in items: + title = str(node.get("title") or "").strip() + if not title: + continue + level = node.get("level") + try: + level_i = int(level) if level is not None else 1 + except (TypeError, ValueError): + level_i = 1 + page = node.get("page") + entry: dict[str, Any] = { + "heading": title, + "level": level_i, + "page": page if page is None else int(page), + } + entries.append(entry) + children = node.get("children") or [] + if isinstance(children, list) and children: + walk([child for child in children if isinstance(child, dict)]) + + walk([node for node in nodes if isinstance(node, dict)]) + return entries + + +def verify_entries( + entries: list[dict[str, Any]], + page_texts: dict[int, str], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Keep entries whose title appears on the pointed physical page. + + Entries without a page (null-page parents) are kept without text checks. + Failures drop only that entry; this is not a gate. + """ + kept: list[dict[str, Any]] = [] + dropped: list[dict[str, Any]] = [] + for entry in entries: + page = entry.get("page") + if page is None: + kept.append(entry) + continue + try: + page_i = int(page) + except (TypeError, ValueError): + dropped.append(entry) + continue + if _title_on_page(str(entry.get("heading") or ""), page_i, page_texts): + kept.append(entry) + else: + dropped.append(entry) + return kept, dropped + + +def build_tree_digest_from_entries( + entries: list[dict[str, Any]], + *, + max_title_chars: int = _DEFAULT_DIGEST_TITLE_CHARS, +) -> str: + """Compact outline digest for a single true/false VLM confirm call.""" + lines: list[str] = [] + for entry in entries: + heading = str(entry.get("heading") or "").strip() + if not heading: + continue + try: + level = int(entry.get("level") or 1) + except (TypeError, ValueError): + level = 1 + indent = " " * max(level - 1, 0) + clipped = ( + heading if len(heading) <= max_title_chars else heading[:max_title_chars] + ) + lines.append(f"{indent}L{level} {clipped}") + return "\n".join(lines) + + +def _title_on_page(title: str, page: int, page_texts: dict[int, str]) -> bool: + needle = _compact(clean_toc_title(title) or title) + if not needle: + return False + haystack = _compact(page_texts.get(page, "")) + return bool(haystack) and needle in haystack + + +def _compact(text: str) -> str: + return re.sub(r"\s+", "", normalize_heading_text(text)).casefold() diff --git a/apps/worker/app/services/document_agent/structure/page_locate_agent.py b/apps/worker/app/services/document_agent/structure/page_locate_agent.py deleted file mode 100644 index f16f02735..000000000 --- a/apps/worker/app/services/document_agent/structure/page_locate_agent.py +++ /dev/null @@ -1,163 +0,0 @@ -"""VLM page verification for page-memory offset calibration. - -This module provides the deterministic-input, VLM-arbitrated helper used by the -page-memory skeleton calibration to confirm which candidate page starts a given -section title. The former residual ReAct sub-agent has been removed; calibration -now drives offset-guided bulk anchoring directly and only needs this verifier. -""" - -from __future__ import annotations - -import base64 -import json -import os -import time -from typing import Any, cast - -from loguru import logger - -from app.services.document_agent.manifest import ToolContext -from app.services.document_agent.structure.hierarchy_locator import TitleMatch - -VLM_CONFIRMED_DEFAULT_CONFIDENCE = 0.75 -GREP_ONLY_CONFIDENCE_CAP = 0.62 -RENDER_FAILED_GREP_CONFIDENCE_CAP = 0.58 -BUDGET_EXHAUSTED_GREP_CONFIDENCE_CAP = 0.56 -VLM_FAILED_GREP_CONFIDENCE_CAP = 0.54 - - -def verify_section_page_choice( - *, - ctx: ToolContext | None, - title: str, - candidate_matches: list[TitleMatch], - candidate_page_cap: int, -) -> dict[str, Any]: - candidates = candidate_matches[: max(candidate_page_cap, 1)] - if not candidates: - return { - "selected_page": None, - "confidence": 0.0, - "source": "agent_heuristic", - "reason": "no grep candidates", - } - - model = None - if ctx is not None: - model = ctx.settings.get("vlm_model") or os.environ.get("IMAGE_MODEL") - pages = [match.page for match in candidates] - if ctx is None or not model or ctx.budget is None: - best = candidates[0] - return { - "selected_page": best.page, - "candidate_pages": pages, - "confidence": min(best.confidence, GREP_ONLY_CONFIDENCE_CAP), - "source": "agent_heuristic", - "reason": "VLM unavailable; selected top grep candidate", - } - - from app.services.document_agent.visual import render_pages - - rendered = render_pages( - ctx, - pages, - folder_name="page_locate_pages", - prefix="locate", - timeout=120, - ) - if not rendered: - best = candidates[0] - return { - "selected_page": best.page, - "candidate_pages": pages, - "confidence": min(best.confidence, RENDER_FAILED_GREP_CONFIDENCE_CAP), - "source": "agent_heuristic", - "reason": "render failed; selected top grep candidate", - } - - prompt = _build_verify_prompt(title=title, candidates=candidates) - est = 800 * len(rendered) + 800 - stage = "calibration" - if not ctx.budget.try_reserve("visual", est, stage=stage): - best = candidates[0] - return { - "selected_page": best.page, - "candidate_pages": pages, - "confidence": min(best.confidence, BUDGET_EXHAUSTED_GREP_CONFIDENCE_CAP), - "source": "agent_heuristic", - "reason": "calibration visual budget exhausted; selected top grep candidate", - } - - content_parts: list[dict[str, Any]] = [{"type": "text", "text": prompt}] - for item in rendered: - with open(str(item["png_path"]), "rb") as image_file: - img_b64 = base64.b64encode(image_file.read()).decode() - content_parts.append({"type": "text", "text": f"\n--- Page {item['page']} ---"}) - content_parts.append( - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{img_b64}"}, - } - ) - - start = time.monotonic() - try: - from shared.services.ai.llm_overrides import get_vision_client - - client, model = get_vision_client(requested_model=model) - raw, usage = client.chat_completion_with_usage( - messages=cast(Any, [{"role": "user", "content": content_parts}]), - model=model, - temperature=0.0, - max_tokens=400, - response_format={"type": "json_object"}, - usage_task="page_memory.page_locate", - ) - ctx.budget.commit( - "visual", - actual=usage.get("total_tokens", est), - est=est, - stage=stage, - ) - payload = json.loads(raw) - selected_page = payload.get("selected_page") - if selected_page is not None: - selected_page = int(selected_page) - if selected_page not in pages: - selected_page = None - return { - "selected_page": selected_page, - "candidate_pages": pages, - "confidence": float(payload.get("confidence") or VLM_CONFIRMED_DEFAULT_CONFIDENCE), - "source": "agent_vlm", - "reason": str(payload.get("reason") or ""), - "latency_ms": int((time.monotonic() - start) * 1000), - "tokens_used": usage.get("total_tokens", 0), - } - except Exception as exc: - ctx.budget.refund("visual", est=est, stage=stage) - best = candidates[0] - logger.warning("[page_locate.agent] VLM failed for title={!r}: {}", title, exc) - return { - "selected_page": best.page, - "candidate_pages": pages, - "confidence": min(best.confidence, VLM_FAILED_GREP_CONFIDENCE_CAP), - "source": "agent_heuristic", - "reason": f"VLM failed ({type(exc).__name__}); selected top grep candidate", - } - - -def _build_verify_prompt(*, title: str, candidates: list[TitleMatch]) -> str: - candidate_lines = "\n".join( - f"- page {match.page}: source={match.source}, line={match.matched_line!r}" - for match in candidates - ) - return ( - "You are a page-location sub-agent for a PDF hierarchy parser.\n" - "Choose which candidate page is the true START page of the section title, " - "not a table-of-contents entry, page header, footer, or body-text mention.\n" - f"Section title: {title!r}\n" - f"Candidates:\n{candidate_lines}\n" - "Return strict JSON: {\"selected_page\": number|null, " - "\"confidence\": number, \"reason\": \"brief explanation\"}." - ) diff --git a/apps/worker/app/services/document_agent/structure/section_page_verify.py b/apps/worker/app/services/document_agent/structure/section_page_verify.py new file mode 100644 index 000000000..aa7d363e3 --- /dev/null +++ b/apps/worker/app/services/document_agent/structure/section_page_verify.py @@ -0,0 +1,91 @@ +"""Section-start page verification via ``inspect.pages`` (Phase-2). + +Confirms whether a candidate physical page is the true START of a TOC title. +Uses the same question schema as Phase-1 forward scan; no heuristic fallback. + +``inspect_pages`` is imported lazily so ``anchoring_primitives`` → this module +does not race ``tools.__init__`` → ``propose_shard_plan`` → anchoring. +""" + +from __future__ import annotations + +from typing import Any + +from app.services.document_agent.calibration.prompts import ( + SECTION_START_ANSWER_KEYS, + build_section_start_question, + coerce_found, + coerce_found_page, +) +from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.structure.hierarchy_locator import TitleMatch + + +def verify_section_page_choice( + *, + ctx: ToolContext | None, + title: str, + candidate_matches: list[TitleMatch], + candidate_page_cap: int, +) -> dict[str, Any]: + candidates = candidate_matches[: max(candidate_page_cap, 1)] + pages = [match.page for match in candidates] + if not candidates: + return { + "selected_page": None, + "candidate_pages": [], + "source": "inspect_vlm", + "reason": "no candidates", + } + if ctx is None: + return { + "selected_page": None, + "candidate_pages": pages, + "source": "inspect_vlm", + "reason": "ctx missing", + } + + from app.services.document_agent.tools.inspect_pages import inspect_pages + + result = inspect_pages( + ctx, + { + "pages": pages, + "page_cap": len(pages), + "question": build_section_start_question(title), + "answer_keys": SECTION_START_ANSWER_KEYS, + "folder_name": "calibration_verify", + "prefix": "verify", + "usage_task": "calibration.verify_section_page", + }, + ) + if result.status != "ok": + return { + "selected_page": None, + "candidate_pages": pages, + "source": "inspect_vlm", + "reason": result.error or "inspect.pages failed", + "latency_ms": result.latency_ms, + "tokens_used": result.tokens_used, + } + + fields = (result.payload or {}).get("fields") or {} + found_page = coerce_found_page(fields.get("found_page"), pages=pages) + found = coerce_found(fields.get("found")) and found_page is not None + if not found: + return { + "selected_page": None, + "candidate_pages": pages, + "source": "inspect_vlm", + "reason": str((result.payload or {}).get("answer") or "not found"), + "latency_ms": result.latency_ms, + "tokens_used": result.tokens_used, + } + return { + "selected_page": found_page, + "candidate_pages": pages, + "source": "inspect_vlm", + "reason": str((result.payload or {}).get("answer") or ""), + "latency_ms": result.latency_ms, + "tokens_used": result.tokens_used, + } diff --git a/apps/worker/app/services/document_agent/structure/structure_anchoring.py b/apps/worker/app/services/document_agent/structure/structure_anchoring.py deleted file mode 100644 index b32a68494..000000000 --- a/apps/worker/app/services/document_agent/structure/structure_anchoring.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Compatibility exports for hierarchy anchoring. - -Low-level anchoring primitives live in :mod:`anchoring_primitives`; the -calibration-owned orchestrator is resolved lazily to keep imports acyclic. -""" - -from __future__ import annotations - -from importlib import import_module -from typing import Any - -from app.services.document_agent.structure import anchoring_primitives as _anchoring -from app.services.document_agent.structure.anchoring_primitives import ( - SkeletonAnchor, - TitleMatch, - TitleNode, -) -from app.services.document_agent.manifest import ToolContext -from app.services.document_agent.structure.page_locate_agent import ( - verify_section_page_choice, -) - -__all__ = [ - "SkeletonAnchor", - "TitleMatch", - "TitleNode", - "anchor_hierarchy", - "anchor_hierarchy_from_offset", - "bulk_offset_matches", - "deserialize_skeleton_anchor", - "deserialize_title_match", - "locate_null_page_parent_overrides", - "offset_guided_anchoring", - "prune_out_of_scope_nodes", - "prune_unanchored_toc_leaves", - "serialize_skeleton_anchor", - "serialize_title_match", - "toc_range_end", - "toc_range_start", -] - -anchor_hierarchy_from_offset = _anchoring.anchor_hierarchy_from_offset -bulk_offset_matches = _anchoring.bulk_offset_matches -deserialize_skeleton_anchor = _anchoring.deserialize_skeleton_anchor -deserialize_title_match = _anchoring.deserialize_title_match -locate_null_page_parent_overrides = _anchoring.locate_null_page_parent_overrides -prune_out_of_scope_nodes = _anchoring.prune_out_of_scope_nodes -prune_unanchored_toc_leaves = _anchoring.prune_unanchored_toc_leaves -serialize_skeleton_anchor = _anchoring.serialize_skeleton_anchor -serialize_title_match = _anchoring.serialize_title_match -toc_range_end = _anchoring.toc_range_end -toc_range_start = _anchoring.toc_range_start - - -def offset_guided_anchoring( - *, - nodes: list[TitleNode], - offset: int, - ctx: ToolContext, - page_count: int, - calibration_overrides: dict[tuple[str, ...], TitleMatch], -) -> dict[tuple[str, ...], TitleMatch] | None: - """Forward phase-2 anchoring while preserving the historical patch seam.""" - original = _anchoring.verify_section_page_choice - _anchoring.verify_section_page_choice = verify_section_page_choice - try: - return _anchoring.offset_guided_anchoring( - nodes=nodes, - offset=offset, - ctx=ctx, - page_count=page_count, - calibration_overrides=calibration_overrides, - ) - finally: - _anchoring.verify_section_page_choice = original - - -def anchor_hierarchy( - *, - nodes: list[TitleNode], - toc_hierarchies: list[dict[str, Any]] | None, - page_texts: dict[int, str], - body_pages: list[int], - page_count: int, - ctx: ToolContext | None, -) -> tuple[list[TitleNode], SkeletonAnchor]: - """Resolve the calibration-owned orchestration entry point on demand.""" - orchestrator = import_module( - "app.services.document_agent.agents.calibration.orchestrator" - ) - return orchestrator.anchor_hierarchy( - nodes=nodes, - toc_hierarchies=toc_hierarchies, - page_texts=page_texts, - body_pages=body_pages, - page_count=page_count, - ctx=ctx, - ) diff --git a/apps/worker/app/services/document_agent/structure/toc_anchoring.py b/apps/worker/app/services/document_agent/structure/toc_anchoring.py index 353280913..74eff2f09 100644 --- a/apps/worker/app/services/document_agent/structure/toc_anchoring.py +++ b/apps/worker/app/services/document_agent/structure/toc_anchoring.py @@ -11,6 +11,7 @@ from app.services.document_agent.manifest import ToolContext from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, + anchor_hierarchy_from_offset, deserialize_skeleton_anchor, deserialize_title_node, serialize_skeleton_anchor, @@ -19,27 +20,32 @@ toc_range_start, ) from app.services.document_agent.structure.hierarchy_locator import ( - ResolvedHierarchyRange, + TitleMatch, TitleNode, collapse_intermediate_single_child_chains, extract_toc_nodes, iter_leaf_title_nodes, - resolve_hierarchy_page_ranges, + normalize_heading_text, ) -_FRONT_TOC_REGION_GAP_PAGES = 5 _LOG_PREFIX = "[profile.toc_anchoring]" +PENDING_TOC_CALIBRATION_CONCURRENCY = 10 def run_toc_anchoring(ctx: ToolContext) -> None: - """Anchor extracted TOC hierarchies onto the profile blackboard.""" - from app.services.document_agent.agents.calibration.orchestrator import ( + """Anchor TOC structure onto the profile blackboard. + + Outline is one route inside this stage: when bookmarks survive self-check and + beat the confirmed printed TOC pages on coverage, anchor from outline with + physical overrides (no calibrate VLM). Otherwise keep the extracted TOC tree + and run the existing VLM calibration path. + """ + from app.services.document_agent.calibration.orchestrator import ( anchor_hierarchy, ) page_count = int(ctx.blackboard.page_count or 0) - hierarchies = list(ctx.blackboard.toc_hierarchies or []) - if page_count <= 0 or not hierarchies: + if page_count <= 0: return page_texts = dict(ctx.blackboard.page_full_text_cache) @@ -47,6 +53,24 @@ def run_toc_anchoring(ctx: ToolContext) -> None: raise ValueError( "page_full_text_cache missing; run text scan before TOC anchoring" ) + + toc_result = ctx.blackboard.toc_result + toc_pages = list(getattr(toc_result, "toc_pages", None) or []) + body_pages = body_pages_excluding_toc(toc_pages, page_count) + + if _try_outline_anchoring_route( + ctx, + page_texts=page_texts, + toc_pages=toc_pages, + body_pages=body_pages, + page_count=page_count, + ): + return + + hierarchies = list(ctx.blackboard.toc_hierarchies or []) + if not hierarchies: + return + filename = Path(ctx.pdf_path).name primary, pending, _summary = select_global_toc_hierarchies( hierarchies=hierarchies, @@ -57,11 +81,6 @@ def run_toc_anchoring(ctx: ToolContext) -> None: return nodes = collapse_intermediate_single_child_chains(nodes) - toc_result = ctx.blackboard.toc_result - body_pages = body_pages_excluding_toc( - getattr(toc_result, "toc_pages", None), - page_count, - ) resolve_nodes, skeleton_anchor = anchor_hierarchy( nodes=nodes, @@ -73,43 +92,207 @@ def run_toc_anchoring(ctx: ToolContext) -> None: ) pending_records: list[dict[str, Any]] = [] if pending: - primary_ranges = resolve_hierarchy_page_ranges( - resolve_nodes, - page_count=page_count, - page_texts=page_texts, - body_pages=body_pages, - match_overrides=skeleton_anchor.match_overrides, - ) - pending_records = _anchor_pending_tocs( + pending_records = _calibrate_pending_tocs( pending_tocs=pending, ctx=ctx, page_texts=page_texts, page_count=page_count, body_pages=body_pages, - primary_ranges=primary_ranges, + ) + _assign_toc_relationships( + root_anchor=skeleton_anchor, + pending_records=pending_records, ) resolve_nodes, skeleton_anchor = _graft_contained_pending( resolve_nodes=resolve_nodes, skeleton_anchor=skeleton_anchor, pending_records=pending_records, page_count=page_count, - page_texts=page_texts, body_pages=body_pages, ) ctx.blackboard.skeleton_anchor = serialize_skeleton_anchor(skeleton_anchor) ctx.blackboard.skeleton_nodes = [ serialize_title_node(node) for node in resolve_nodes ] - ctx.blackboard.toc_page_offset = skeleton_anchor.offset ctx.blackboard.pending_skeleton_anchors = pending_records +def _try_outline_anchoring_route( + ctx: ToolContext, + *, + page_texts: dict[int, str], + toc_pages: list[int], + body_pages: list[int], + page_count: int, +) -> bool: + """Return True when outline wins and skeleton state has been written. + + Consumes ``blackboard.pdf_outline_roots`` written during the find-stage + ``probe.outline`` call; does not re-open the PDF here. + """ + from app.services.document_agent.structure.outline_check import ( + build_tree_digest_from_entries, + flatten_outline_entries, + verify_entries, + ) + from app.services.document_agent.tools.judge_toc_source import ( + OUTLINE_CHOICE, + judge_toc_source, + ) + + roots = list(ctx.blackboard.pdf_outline_roots or []) + if not roots: + return False + + kept, _dropped = verify_entries( + flatten_outline_entries(roots), + page_texts, + ) + paged_kept = [entry for entry in kept if entry.get("page") is not None] + if not paged_kept: + return False + + if toc_pages: + judge_result = judge_toc_source( + ctx, + { + "outline_digest": build_tree_digest_from_entries(kept), + "toc_pages": toc_pages, + }, + ) + if judge_result.status != "ok": + return False + if (judge_result.payload or {}).get("choice") != OUTLINE_CHOICE: + return False + + toc_with_level: list[dict[str, Any]] = [] + for entry in kept: + row: dict[str, Any] = { + "heading": entry["heading"], + "level": entry["level"], + } + if entry.get("page") is not None: + row["physical_page"] = int(entry["page"]) + toc_with_level.append(row) + + hierarchy = { + "source": "pdf_outline", + "toc_with_level": toc_with_level, + } + if not _write_outline_skeleton( + ctx, + hierarchies=[hierarchy], + page_texts=page_texts, + body_pages=body_pages, + page_count=page_count, + ): + return False + + ctx.blackboard.toc_hierarchies = [hierarchy] + if ctx.blackboard.toc_result is not None: + ctx.blackboard.toc_result.method = "pdf_outline" + ctx.blackboard.toc_result.notes = ( + "outline won coverage compare at toc anchoring; " + f"kept confirmed toc_pages={toc_pages}" + ) + return True + + +def _write_outline_skeleton( + ctx: ToolContext, + *, + hierarchies: list[dict[str, Any]], + page_texts: dict[int, str], + body_pages: list[int], + page_count: int, +) -> bool: + """Anchor outline rows via physical overrides (no calibrate VLM). + + Trust outline tree shape: do not collapse single-child chains (VLM path still + collapses). Path keys must stay aligned with ``outline_physical_overrides``. + """ + nodes = extract_toc_nodes(hierarchies) + if not nodes: + return False + overrides = outline_physical_overrides(hierarchies) + resolve_nodes, skeleton_anchor = anchor_hierarchy_from_offset( + nodes=nodes, + offset_hint=0, + calibration_overrides=overrides, + page_texts=page_texts, + body_pages=body_pages, + page_count=page_count, + ctx=ctx, + ) + skeleton_anchor = replace(skeleton_anchor, source="pdf_outline") + ctx.blackboard.skeleton_anchor = serialize_skeleton_anchor(skeleton_anchor) + ctx.blackboard.skeleton_nodes = [ + serialize_title_node(node) for node in resolve_nodes + ] + ctx.blackboard.pending_skeleton_anchors = [] + logger.info( + "{} outline anchoring: overrides={} pruned={} nodes={}", + _LOG_PREFIX, + len(skeleton_anchor.match_overrides), + skeleton_anchor.pruned_count, + len(resolve_nodes), + ) + return True + + +def outline_physical_overrides( + hierarchies: list[dict[str, Any]], +) -> dict[tuple[str, ...], TitleMatch]: + """Build path→physical TitleMatch from outline rows carrying ``physical_page``.""" + overrides: dict[tuple[str, ...], TitleMatch] = {} + for hierarchy in hierarchies: + entries = hierarchy.get("toc_with_level") or [] + if not isinstance(entries, list): + continue + stack: list[tuple[int, str]] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + title = normalize_heading_text(str(entry.get("heading") or "")) + if not title or len(title) < 2: + continue + try: + level = int(entry.get("level") or 1) + except (TypeError, ValueError): + level = 1 + while stack and stack[-1][0] >= level: + stack.pop() + stack.append((level, title)) + path = tuple(item[1] for item in stack) + raw_page = entry.get("physical_page") + if raw_page is None: + continue + try: + page = int(raw_page) + except (TypeError, ValueError): + continue + if page < 1: + continue + overrides[path] = TitleMatch( + page=page, + source="pdf_outline", + matched_line="", + candidates=[page], + ) + return overrides + + def select_global_toc_hierarchies( *, hierarchies: list[dict[str, Any]], filename: str, ) -> tuple[list[dict[str, Any]] | None, list[dict[str, Any]], dict[str, Any]]: - """Split TOC hierarchies into primary (front cluster) and pending.""" + """Split TOC hierarchies into earliest forest root and the rest. + + The earliest TOC is only the serialization root for ``skeleton_nodes``. + Relationship classify treats every calibrated TOC as equal peers on + evidenced physical spans. + """ if len(hierarchies) <= 1: return (hierarchies or None), [], {} @@ -126,48 +309,27 @@ def select_global_toc_hierarchies( enumerate(hierarchies), key=lambda item: toc_range_start(item[1]) or 0, ) - selected_indices: set[int] = set() - pending_indices: list[int] = [] - cluster_end: int | None = None - - for original_index, hierarchy in sorted_items: - start = toc_range_start(hierarchy) - end = toc_range_end(hierarchy) - if start is None or end is None: - selected_indices.add(original_index) - continue - if cluster_end is None: - selected_indices.add(original_index) - cluster_end = end - continue - if start <= cluster_end + _FRONT_TOC_REGION_GAP_PAGES: - selected_indices.add(original_index) - cluster_end = max(cluster_end, end) - continue - pending_indices.append(original_index) + primary_index = sorted_items[0][0] + pending_indices = [index for index, _hierarchy in sorted_items[1:]] - selected = [ - hierarchy - for index, hierarchy in enumerate(hierarchies) - if index in selected_indices - ] + selected = [hierarchies[primary_index]] pending = [hierarchies[i] for i in pending_indices] if pending: logger.info( - "{} toc split: primary={} pending={} filename={}", + "{} toc split: root_forest={} peers={} filename={}", _LOG_PREFIX, len(selected), len(pending), filename, ) summary = { - "strategy": "front_cluster_with_pending", + "strategy": "earliest_forest_rest_pending", "input_count": len(hierarchies), "primary_count": len(selected), "pending_count": len(pending), } - return (selected or None), pending, summary + return selected, pending, summary def body_pages_excluding_toc(toc_pages: Any, page_count: int) -> list[int]: @@ -200,142 +362,238 @@ def pending_toc_body_scope( return toc_scope_end, toc_body_pages +def evidenced_physical_span( + match_overrides: dict[tuple[str, ...], TitleMatch], +) -> tuple[int, int] | None: + """Min/max physical pages from overrides; no end-of-document extrapolation.""" + pages = [match.page for match in match_overrides.values() if match.page is not None] + if not pages: + return None + return min(pages), max(pages) + + +def find_tightest_containing_host( + candidate_span: tuple[int, int], + hosts: list[tuple[Any, tuple[int, int]]], +) -> Any | None: + """Return host key whose evidenced span tightly covers ``candidate_span``.""" + best_key: Any | None = None + best_size: int | None = None + c0, c1 = candidate_span + if c0 > c1: + return None + for key, (h0, h1) in hosts: + if h0 <= c0 and c1 <= h1 and (h0, h1) != (c0, c1): + size = h1 - h0 + if best_size is None or size < best_size: + best_size = size + best_key = key + return best_key + + def classify_toc_relationship( *, - offset: int, - nodes: list[TitleNode], - primary_ranges: list[ResolvedHierarchyRange], - page_count: int, + candidate_span: tuple[int, int] | None, + host_spans: list[tuple[int, int]], ) -> str: - """Classify a pending TOC as parallel or contained vs primary ranges. + """Classify a TOC unit vs peer evidenced physical spans. - parallel: the pending TOC covers pages beyond the primary tree's *anchored* - content (i.e. the last explicitly-located section start page). - contained: the pending TOC's content falls strictly within a primary - section's explicitly-anchored range. - """ - leaves = [ - node - for _, node in iter_leaf_title_nodes(nodes) - if node.printed_page is not None - ] - if not leaves: - return "unresolvable" - - first_printed = leaves[0].printed_page - last_printed = leaves[-1].printed_page - if first_printed is None or last_printed is None: - return "unresolvable" - first_physical = first_printed + offset - last_physical = last_printed + offset + contained: some peer span fully covers the candidate span + parallel: no peer covers it + unresolvable: candidate has no evidenced pages - if first_physical < 1 or first_physical > page_count: + Spans are min/max override pages only — never extrapolated to document end. + """ + if candidate_span is None: return "unresolvable" - - if not primary_ranges: + hosts = [(index, span) for index, span in enumerate(host_spans)] + if find_tightest_containing_host(candidate_span, hosts) is None: return "parallel" + return "contained" - # Use the last *start_page* among primary ranges as the boundary of - # explicitly-anchored content. The end_page of the last section is often - # extended to page_count by default and doesn't reflect real content coverage. - last_anchored_start = max( - (r.start_page for r in primary_ranges if r.start_page is not None), default=0 - ) - if first_physical > last_anchored_start: - return "parallel" - - min_level = min(r.level for r in primary_ranges) - top_level_ranges = [r for r in primary_ranges if r.level == min_level] - for r in top_level_ranges: - if r.start_page and r.end_page: - if r.start_page <= first_physical and last_physical <= r.end_page: - return "contained" +def _assign_toc_relationships( + *, + root_anchor: SkeletonAnchor, + pending_records: list[dict[str, Any]], +) -> None: + """Equal-footing classify: every pending vs root + other pendings' spans.""" + root_span = evidenced_physical_span(root_anchor.match_overrides) + keyed_spans: list[tuple[Any, tuple[int, int] | None]] = [("root", root_span)] + for index, record in enumerate(pending_records): + if record.get("relationship") == "unresolvable": + keyed_spans.append((index, None)) + continue + anchor_raw = record.get("skeleton_anchor") + if not isinstance(anchor_raw, dict): + record["relationship"] = "unresolvable" + keyed_spans.append((index, None)) + continue + span = evidenced_physical_span( + deserialize_skeleton_anchor(anchor_raw).match_overrides + ) + keyed_spans.append((index, span)) + if span is None: + record["relationship"] = "unresolvable" - return "parallel" + for index, record in enumerate(pending_records): + if record.get("relationship") == "unresolvable": + record.pop("host", None) + continue + candidate = keyed_spans[index + 1][1] + if candidate is None: + record["relationship"] = "unresolvable" + record.pop("host", None) + continue + hosts = [ + (key, span) + for key, span in keyed_spans + if key != index and span is not None + ] + host_key = find_tightest_containing_host(candidate, hosts) + if host_key is None: + record["relationship"] = "parallel" + record.pop("host", None) + else: + record["relationship"] = "contained" + record["host"] = host_key + logger.info( + "{} peer classify toc_range={} relationship={} host={} span={}", + _LOG_PREFIX, + (record.get("toc") or {}).get("toc_range"), + record["relationship"], + record.get("host"), + candidate, + ) -def _anchor_pending_tocs( +def _calibrate_one_pending_toc( *, + index: int, + pending_toc: dict[str, Any], pending_tocs: list[dict[str, Any]], ctx: ToolContext, page_texts: dict[int, str], page_count: int, body_pages: list[int], - primary_ranges: list[ResolvedHierarchyRange], -) -> list[dict[str, Any]]: - from app.services.document_agent.agents.calibration.procedure import ( +) -> dict[str, Any] | None: + from app.services.document_agent.calibration.procedure import ( finalize_calibration_result, pick_primary_offset, ) - from app.services.document_agent.agents.calibration.service import ( + from app.services.document_agent.calibration.service import ( calibrate_offset, ) - records: list[dict[str, Any]] = [] - for i, pending_toc in enumerate(pending_tocs): - nodes = extract_toc_nodes([pending_toc]) - if not nodes: - continue - nodes = collapse_intermediate_single_child_chains(nodes) - toc_scope_end, toc_body_pages = pending_toc_body_scope( - pending_tocs=pending_tocs, - index=i, - page_count=page_count, - body_pages=body_pages, + nodes = extract_toc_nodes([pending_toc]) + if not nodes: + return None + nodes = collapse_intermediate_single_child_chains(nodes) + toc_scope_end, toc_body_pages = pending_toc_body_scope( + pending_tocs=pending_tocs, + index=index, + page_count=page_count, + body_pages=body_pages, + ) + phase1 = calibrate_offset( + toc_hierarchies=[pending_toc], + ctx=ctx, + page_texts=page_texts, + page_count=toc_scope_end, + ) + offset = pick_primary_offset(phase1) + if offset is None: + logger.info( + "{} pending TOC toc_range={}: calibration failed, skipping", + _LOG_PREFIX, + pending_toc.get("toc_range"), ) - phase1 = calibrate_offset( - nodes=nodes, - toc_hierarchies=[pending_toc], - ctx=ctx, - page_texts=page_texts, - page_count=toc_scope_end, + return None + if not any( + node.printed_page is not None + for _path, node in iter_leaf_title_nodes(nodes) + ): + logger.info( + "{} pending TOC toc_range={}: no printed pages, unresolvable", + _LOG_PREFIX, + pending_toc.get("toc_range"), ) - offset = pick_primary_offset(phase1) - if offset is None: - logger.info( - "{} pending TOC toc_range={}: calibration failed, skipping", - _LOG_PREFIX, - pending_toc.get("toc_range"), - ) - continue - relationship = classify_toc_relationship( - offset=offset, - nodes=nodes, - primary_ranges=primary_ranges, - page_count=page_count, + return { + "toc": pending_toc, + "relationship": "unresolvable", + } + resolve_nodes, skeleton_anchor, _finalized = finalize_calibration_result( + result=phase1, + entries=list(pending_toc.get("toc_with_level") or []), + toc_hierarchies=[pending_toc], + ctx=ctx, + page_count=toc_scope_end, + page_texts=page_texts, + body_pages=toc_body_pages, + nodes=nodes, + ) + if evidenced_physical_span(skeleton_anchor.match_overrides) is None: + logger.info( + "{} pending TOC toc_range={}: no evidenced pages, unresolvable", + _LOG_PREFIX, + pending_toc.get("toc_range"), ) - if relationship == "unresolvable": - logger.info( - "{} pending TOC toc_range={}: unresolvable, skipping", - _LOG_PREFIX, - pending_toc.get("toc_range"), - ) - records.append( - { - "toc": pending_toc, - "relationship": relationship, - } - ) - continue - resolve_nodes, skeleton_anchor, _finalized = finalize_calibration_result( - result=phase1, - entries=list(pending_toc.get("toc_with_level") or []), - toc_hierarchies=[pending_toc], + return { + "toc": pending_toc, + "relationship": "unresolvable", + } + return { + "toc": pending_toc, + "nodes": [serialize_title_node(node) for node in resolve_nodes], + "skeleton_anchor": serialize_skeleton_anchor(skeleton_anchor), + } + + +def _calibrate_pending_tocs( + *, + pending_tocs: list[dict[str, Any]], + ctx: ToolContext, + page_texts: dict[int, str], + page_count: int, + body_pages: list[int], +) -> list[dict[str, Any]]: + if not pending_tocs: + return [] + + from gevent.pool import Pool as GeventPool + + pool_size = min(PENDING_TOC_CALIBRATION_CONCURRENCY, len(pending_tocs)) + logger.info( + "{} pending TOC calibration: count={} concurrency={}", + _LOG_PREFIX, + len(pending_tocs), + pool_size, + ) + pool = GeventPool(size=pool_size) + jobs = [ + pool.spawn( + _calibrate_one_pending_toc, + index=i, + pending_toc=pending_toc, + pending_tocs=pending_tocs, ctx=ctx, - page_count=toc_scope_end, page_texts=page_texts, - body_pages=toc_body_pages, - nodes=nodes, - ) - records.append( - { - "toc": pending_toc, - "relationship": relationship, - "nodes": [serialize_title_node(node) for node in resolve_nodes], - "skeleton_anchor": serialize_skeleton_anchor(skeleton_anchor), - } + page_count=page_count, + body_pages=body_pages, ) + for i, pending_toc in enumerate(pending_tocs) + ] + pool.join() + + records: list[dict[str, Any]] = [] + for job in jobs: + try: + record = job.get() + except Exception as exc: + logger.warning("{} pending TOC calibration job failed: {}", _LOG_PREFIX, exc) + continue + if record is not None: + records.append(record) return records @@ -345,14 +603,13 @@ def _graft_contained_pending( skeleton_anchor: SkeletonAnchor, pending_records: list[dict[str, Any]], page_count: int, - page_texts: dict[int, str], body_pages: list[int], ) -> tuple[list[TitleNode], SkeletonAnchor]: from app.services.document_agent.structure.toc_graft import graft_contained_toc - nodes = resolve_nodes - overrides = dict(skeleton_anchor.match_overrides) - for record in pending_records: + pending_forests: dict[int, tuple[list[TitleNode], dict[tuple[str, ...], TitleMatch]]] = {} + contained_order: list[tuple[int, int]] = [] + for index, record in enumerate(pending_records): if record.get("relationship") != "contained": continue nodes_raw = record.get("nodes") or [] @@ -365,17 +622,86 @@ def _graft_contained_pending( if not contained_nodes: continue contained_anchor = deserialize_skeleton_anchor(anchor_raw) + span = evidenced_physical_span(contained_anchor.match_overrides) + if span is None: + continue + pending_forests[index] = ( + contained_nodes, + dict(contained_anchor.match_overrides), + ) + contained_order.append((span[1] - span[0], index)) + + for index, record in enumerate(pending_records): + if index in pending_forests: + continue + if record.get("relationship") not in {"parallel", "contained"}: + continue + nodes_raw = record.get("nodes") or [] + anchor_raw = record.get("skeleton_anchor") + if not isinstance(anchor_raw, dict) or not nodes_raw: + continue + host_nodes = [ + deserialize_title_node(node) for node in nodes_raw if isinstance(node, dict) + ] + if not host_nodes: + continue + pending_forests[index] = ( + host_nodes, + dict(deserialize_skeleton_anchor(anchor_raw).match_overrides), + ) + + contained_order.sort() + nodes = resolve_nodes + overrides = dict(skeleton_anchor.match_overrides) + + for _size, index in contained_order: + record = pending_records[index] + forest = pending_forests.get(index) + if forest is None: + continue + contained_nodes, contained_overrides = forest + host = record.get("host") + if host == "root": + grafted = graft_contained_toc( + primary_nodes=nodes, + primary_overrides=overrides, + contained_nodes=contained_nodes, + contained_overrides=contained_overrides, + page_count=page_count, + body_pages=body_pages, + ) + nodes = grafted.nodes + overrides = grafted.match_overrides + record["grafted"] = True + record["graft"] = grafted.events + continue + if not isinstance(host, int) or host not in pending_forests: + record["relationship"] = "parallel" + record.pop("host", None) + record.pop("grafted", None) + record.pop("graft", None) + continue + host_nodes, host_overrides = pending_forests[host] grafted = graft_contained_toc( - primary_nodes=nodes, - primary_overrides=overrides, + primary_nodes=host_nodes, + primary_overrides=host_overrides, contained_nodes=contained_nodes, - contained_overrides=contained_anchor.match_overrides, + contained_overrides=contained_overrides, page_count=page_count, - page_texts=page_texts, body_pages=body_pages, ) - nodes = grafted.nodes - overrides = grafted.match_overrides + pending_forests[host] = (grafted.nodes, grafted.match_overrides) + host_record = pending_records[host] + host_record["nodes"] = [ + serialize_title_node(node) for node in grafted.nodes + ] + host_anchor = deserialize_skeleton_anchor( + host_record.get("skeleton_anchor") or {} + ) + host_record["skeleton_anchor"] = serialize_skeleton_anchor( + replace(host_anchor, match_overrides=grafted.match_overrides) + ) record["grafted"] = True record["graft"] = grafted.events + return nodes, replace(skeleton_anchor, match_overrides=overrides) diff --git a/apps/worker/app/services/document_agent/structure/toc_graft.py b/apps/worker/app/services/document_agent/structure/toc_graft.py index efebf6955..be8991ce2 100644 --- a/apps/worker/app/services/document_agent/structure/toc_graft.py +++ b/apps/worker/app/services/document_agent/structure/toc_graft.py @@ -8,9 +8,10 @@ from loguru import logger from app.services.document_agent.structure.hierarchy_locator import ( - ResolvedHierarchyRange, TitleMatch, TitleNode, + coverage_by_path, + deepest_covering_path, resolve_hierarchy_page_ranges, ) @@ -31,7 +32,6 @@ def graft_contained_toc( contained_nodes: list[TitleNode], contained_overrides: dict[tuple[str, ...], TitleMatch], page_count: int, - page_texts: dict[int, str], body_pages: list[int], ) -> ContainedGraftResult: """Merge one contained TOC forest into the current primary tree.""" @@ -40,11 +40,10 @@ def graft_contained_toc( ranges = resolve_hierarchy_page_ranges( primary_nodes, page_count=page_count, - page_texts=page_texts, body_pages=body_pages, match_overrides=overrides, ) - coverage = _coverage_by_path(ranges) + coverage = coverage_by_path(ranges) nodes = _graft_children( parent=None, parent_path=(), @@ -59,24 +58,6 @@ def graft_contained_toc( return ContainedGraftResult(nodes=nodes, match_overrides=overrides, events=events) -def _coverage_by_path( - ranges: list[ResolvedHierarchyRange], -) -> dict[tuple[str, ...], tuple[int, int]]: - coverage: dict[tuple[str, ...], tuple[int, int]] = {} - for item in ranges: - for depth in range(1, len(item.path_titles) + 1): - path = item.path_titles[:depth] - span = coverage.get(path) - if span is None: - coverage[path] = (item.start_page, item.end_page) - else: - coverage[path] = ( - min(span[0], item.start_page), - max(span[1], item.end_page), - ) - return coverage - - def _graft_children( *, parent: TitleNode | None, @@ -160,13 +141,8 @@ def _graft_one_child( "title_equal": matched.title == child.title, } ) - _remap_overrides( - contained_overrides=contained_overrides, - primary_overrides=primary_overrides, - old_prefix=contained_path, - new_prefix=matched_path, - drop_root=True, - ) + # Do not remap descendant overrides here — only successfully + # attached/deduped children may write into primary_overrides. new_matched = replace( matched, children=_graft_children( @@ -199,26 +175,38 @@ def _graft_one_child( contained_overrides=contained_overrides, events=events, ) + reason = ( + "outside_parent_coverage" + if span is not None + else "missing_parent_coverage" + ) + _record_skip( + events=events, + contained_path=contained_path, + start=start, + reason=reason, + parent_path=parent_path, + parent_span=span, + ) return primary_children - covering = _longest_covering_path(coverage, start) + covering = deepest_covering_path(coverage, start=start, end=start) if covering is None: - events.append( - { - "action": "skip", - "contained_path": contained_path, - "start": start, - } + _record_skip( + events=events, + contained_path=contained_path, + start=start, + reason="no_covering_path", ) return primary_children covering_node = _node_at_path(primary_children, covering) if covering_node is None: - events.append( - { - "action": "skip", - "contained_path": contained_path, - "start": start, - } + _record_skip( + events=events, + contained_path=contained_path, + start=start, + reason="covering_node_missing", + parent_path=covering, ) return primary_children updated_parent = replace( @@ -291,6 +279,28 @@ def _sibling_hits( return hits +def _record_skip( + *, + events: list[dict[str, Any]], + contained_path: tuple[str, ...], + start: int, + reason: str, + parent_path: tuple[str, ...] | None = None, + parent_span: tuple[int, int] | None = None, +) -> None: + event: dict[str, Any] = { + "action": "skip", + "contained_path": contained_path, + "start": start, + "reason": reason, + } + if parent_path is not None: + event["parent_path"] = parent_path + if parent_span is not None: + event["parent_span"] = list(parent_span) + events.append(event) + + def _remap_overrides( *, contained_overrides: dict[tuple[str, ...], TitleMatch], @@ -337,19 +347,6 @@ def _insert_by_start( return updated -def _longest_covering_path( - coverage: dict[tuple[str, ...], tuple[int, int]], - start: int, -) -> tuple[str, ...] | None: - covering: tuple[str, ...] | None = None - for path, span in coverage.items(): - if span[0] <= start <= span[1] and ( - covering is None or len(path) > len(covering) - ): - covering = path - return covering - - def _node_at_path(nodes: list[TitleNode], path: tuple[str, ...]) -> TitleNode | None: current: list[TitleNode] = nodes node: TitleNode | None = None diff --git a/apps/worker/app/services/document_agent/structure/toc_link_enrichment.py b/apps/worker/app/services/document_agent/structure/toc_link_enrichment.py deleted file mode 100644 index f82e97ded..000000000 --- a/apps/worker/app/services/document_agent/structure/toc_link_enrichment.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Attach TOC-page hyperlinks onto VLM entries before calibration. - -PROFILE calls this after ``extract.toc_with_boundaries`` and before -``run_toc_anchoring``. Only runs when TOC pages actually contain internal -page hyperlinks (``page.get_links()``). - -Page-number convention (all 1-based after normalize): - - ``get_links()`` dest ``page``: already 1-based — do not add 1. - - VLM / probe pages: already 1-based. - - TODO(bookmarks): ``get_toc()`` ``meta.page`` is 0-based — add 1 when wired. - -Matching (strict): - - Walk VLM ``toc_with_level`` entries in order, once each. - - ``heading.strip() in anchor_text.strip()``. - - Attach only when exactly one link hits; zero or many → leave unmatched. - - Cross-line / truncated anchors are not special-cased. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -from loguru import logger - - -@dataclass(frozen=True) -class TocPageLink: - toc_page: int # 1-based - dest_physical_page: int # 1-based - anchor_text: str - kind: int | None = None - - -@dataclass(frozen=True) -class TocLinkEnrichStats: - toc_pages_scanned: list[int] - links_raw: int - links_internal: int - entries_total: int - entries_matched: int - skipped_no_links: bool = False - - -def _toc_pages_from_hierarchy(hierarchy: dict[str, Any]) -> list[int]: - """Physical pages that are actual TOC content (not VLM scan expansion).""" - pages: set[int] = set() - toc_range = hierarchy.get("toc_range") - if isinstance(toc_range, (list, tuple)) and len(toc_range) >= 2: - start, end = int(toc_range[0]), int(toc_range[1]) - if start > 0 and end >= start: - pages.update(range(start, end + 1)) - # Do NOT include scan_range: that window often covers non-TOC body pages - # used only for VLM boundary detection. - return sorted(pages) - - -def _anchor_text_for_rect(page: Any, rect: Any) -> str: - import fitz - - words = page.get_text("words") or [] - hit: list[tuple[float, float, str]] = [] - target = fitz.Rect(rect) - # Slightly expand so thin link boxes still catch title glyphs. - target = target + (-2, -2, 2, 2) - for word in words: - x0, y0, x1, y1, text = word[:5] - if not str(text).strip(): - continue - if fitz.Rect(x0, y0, x1, y1).intersects(target): - hit.append((float(y0), float(x0), str(text))) - hit.sort() - return " ".join(part for _, _, part in hit).strip() - - -def _link_dest_physical_page(raw_page: Any) -> int: - """Normalize ``page.get_links()`` destination to 1-based physical page. - - PyMuPDF page hyperlinks expose ``link["page"]`` already as 1-based (int or - digit string). Do **not** add 1 here — that off-by-one sent every TOC link - one page past the real click target. - - TODO(bookmarks): ``doc.get_toc()`` outline ``meta.page`` is 0-based. When - bookmark signal is wired into calibration, convert that field with ``+1`` - (or use get_toc's 1-based display page) before merging with links / VLM. - """ - return int(raw_page) - - -def collect_toc_page_links(pdf_path: str, toc_pages: list[int]) -> list[TocPageLink]: - """Collect internal page hyperlinks on TOC pages with nearby anchor text.""" - import fitz - - if not toc_pages: - return [] - - out: list[TocPageLink] = [] - doc = fitz.open(pdf_path) - try: - for toc_page in toc_pages: - if toc_page < 1 or toc_page > doc.page_count: - continue - page = doc[toc_page - 1] - for link in page.get_links() or []: - # Page hyperlinks only (NAMED/GOTO via get_links). Not bookmarks. - dest_raw = link.get("page") - if dest_raw is None: - continue - try: - dest_physical = _link_dest_physical_page(dest_raw) - except (TypeError, ValueError): - continue - if dest_physical < 1 or dest_physical > doc.page_count: - continue - rect = link.get("from") - if rect is None: - continue - anchor = _anchor_text_for_rect(page, rect) - if not anchor: - continue - kind = link.get("kind") - out.append( - TocPageLink( - toc_page=toc_page, - dest_physical_page=dest_physical, - anchor_text=anchor, - kind=int(kind) if kind is not None else None, - ) - ) - finally: - doc.close() - return out - - -def match_toc_entries_to_links( - entries: list[dict[str, Any]], - links: list[TocPageLink], -) -> tuple[list[dict[str, Any]], int]: - """Attach ``link`` when heading.strip() is in exactly one link anchor.""" - matched = 0 - enriched: list[dict[str, Any]] = [] - - for entry in entries: - if not isinstance(entry, dict): - continue - new_entry = { - "heading": entry.get("heading"), - "level": entry.get("level"), - "page_number": entry.get("page_number"), - } - for key, value in entry.items(): - if key in new_entry or key == "link": - continue - new_entry[key] = value - - heading = str(entry.get("heading") or "").strip() - if not heading or not links: - enriched.append(new_entry) - continue - - hits = [ - link - for link in links - if heading in str(link.anchor_text or "").strip() - ] - if len(hits) != 1: - enriched.append(new_entry) - continue - - new_entry["link"] = { - "physical_page": hits[0].dest_physical_page, - } - matched += 1 - enriched.append(new_entry) - - return enriched, matched - - -def enrich_toc_hierarchies_with_links( - *, - pdf_path: str, - toc_hierarchies: list[dict[str, Any]] | None, -) -> tuple[list[dict[str, Any]], TocLinkEnrichStats]: - """Attach optional ``link`` fields onto matching TOC entries. - - If TOC pages have no internal links, hierarchies are returned unchanged. - """ - hierarchies = [dict(h) for h in (toc_hierarchies or []) if isinstance(h, dict)] - if not hierarchies: - return [], TocLinkEnrichStats( - toc_pages_scanned=[], - links_raw=0, - links_internal=0, - entries_total=0, - entries_matched=0, - skipped_no_links=True, - ) - - toc_pages: list[int] = [] - seen: set[int] = set() - for hierarchy in hierarchies: - for page in _toc_pages_from_hierarchy(hierarchy): - if page not in seen: - seen.add(page) - toc_pages.append(page) - - links = collect_toc_page_links(pdf_path, toc_pages) - if not links: - logger.info( - "[toc_link_enrich] no internal links on TOC pages {}; skip", - toc_pages, - ) - return hierarchies, TocLinkEnrichStats( - toc_pages_scanned=toc_pages, - links_raw=0, - links_internal=0, - entries_total=sum( - len(h.get("toc_with_level") or []) - for h in hierarchies - if isinstance(h.get("toc_with_level"), list) - ), - entries_matched=0, - skipped_no_links=True, - ) - - total_entries = 0 - total_matched = 0 - out: list[dict[str, Any]] = [] - for hierarchy in hierarchies: - entries = hierarchy.get("toc_with_level") - if not isinstance(entries, list): - out.append(hierarchy) - continue - enriched_entries, matched = match_toc_entries_to_links(entries, links) - total_entries += len(enriched_entries) - total_matched += matched - new_hierarchy = dict(hierarchy) - new_hierarchy["toc_with_level"] = enriched_entries - out.append(new_hierarchy) - - stats = TocLinkEnrichStats( - toc_pages_scanned=toc_pages, - links_raw=len(links), - links_internal=len(links), - entries_total=total_entries, - entries_matched=total_matched, - skipped_no_links=False, - ) - logger.info( - "[toc_link_enrich] toc_pages={} links={} entries={}/{} matched", - toc_pages, - len(links), - total_matched, - total_entries, - ) - return out, stats diff --git a/apps/worker/app/services/document_agent/tools/__init__.py b/apps/worker/app/services/document_agent/tools/__init__.py index 6ccb1a8a7..bc94a8fda 100644 --- a/apps/worker/app/services/document_agent/tools/__init__.py +++ b/apps/worker/app/services/document_agent/tools/__init__.py @@ -5,7 +5,11 @@ from . import extract_toc_with_boundaries as extract_toc_with_boundaries # noqa: F401 from . import find_toc_anchor_pages as find_toc_anchor_pages # noqa: F401 from . import grep_text as grep_text # noqa: F401 +from . import inspect_pages as inspect_pages # noqa: F401 +from . import judge_toc_source as judge_toc_source # noqa: F401 from . import ocr_pages as ocr_pages # noqa: F401 +from . import probe_links as probe_links # noqa: F401 +from . import probe_outline as probe_outline # noqa: F401 from . import propose_shard_plan as propose_shard_plan # noqa: F401 from . import validate_anatomy_map as validate_anatomy_map # noqa: F401 from . import verdict as verdict # noqa: F401 diff --git a/apps/worker/app/services/document_agent/tools/classify_page_kinds.py b/apps/worker/app/services/document_agent/tools/classify_page_kinds.py index d3a11d52b..53ce492db 100644 --- a/apps/worker/app/services/document_agent/tools/classify_page_kinds.py +++ b/apps/worker/app/services/document_agent/tools/classify_page_kinds.py @@ -15,10 +15,18 @@ def _label_feature(feature: PageFeature) -> PageLabel: return PageLabel( page=page, kind="landscape", - confidence=0.78, - evidence={"width": feature.width, "height": feature.height}, + evidence={ + "source": "geometry", + "rule": "width_gt_height", + "width": feature.width, + "height": feature.height, + }, ) - return PageLabel(page=page, kind="normal", confidence=0.65, evidence={}) + return PageLabel( + page=page, + kind="normal", + evidence={"source": "geometry", "rule": "width_le_height"}, + ) def classify_page_kinds(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult: @@ -36,7 +44,6 @@ def classify_page_kinds(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult: samples[label.kind].append( { "page": label.page, - "confidence": label.confidence, "evidence": label.evidence, "raw_text_length": feature.raw_text_length if feature else None, } diff --git a/apps/worker/app/services/document_agent/tools/extract_toc_with_boundaries.py b/apps/worker/app/services/document_agent/tools/extract_toc_with_boundaries.py index 1480eb4d8..db48fae36 100644 --- a/apps/worker/app/services/document_agent/tools/extract_toc_with_boundaries.py +++ b/apps/worker/app/services/document_agent/tools/extract_toc_with_boundaries.py @@ -10,8 +10,6 @@ from pathlib import Path from typing import Any, cast -from shared.utils.token_estimate import estimate_tokens - from app.services.document_agent.manifest import ( TocAnchorPage, TocEvidence, @@ -22,6 +20,7 @@ from app.services.document_agent.registry import register_tool from app.services.document_agent.tools.vlm_toc_extractor import ( TOC_VLM_MAX_TOKENS, + BatchPageResult, vlm_entries_to_toc_hierarchies, ) from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( @@ -33,12 +32,10 @@ # -- Constants ----------------------------------------------------------------- BOUNDARY_STEP_PAGES = 5 +TOC_VLM_CONCURRENCY = 10 MAX_BOUNDARY_ROUNDS = 6 MAX_TOC_PAGES = BOUNDARY_STEP_PAGES * MAX_BOUNDARY_ROUNDS # 30 -_CONFIRM_STAGE = "toc_confirm" -_CONFIRM_TOKENS_PER_PAGE = 800 - _CONFIRM_PROMPT = ( "You are a document structure analysis expert. " "Below are screenshot(s) of candidate pages extracted from a PDF. " @@ -137,19 +134,9 @@ def _evidence_from_confirm_items( is_toc_start = bool(item.get("is_toc_start")) if is_toc_start: confirmed_pages.add(page) - raw_confidence = item.get("confidence") - try: - confidence = ( - float(raw_confidence) - if raw_confidence is not None - else (0.95 if is_toc_start else 0.05) - ) - except (TypeError, ValueError): - confidence = 0.95 if is_toc_start else 0.05 evidence_by_page[page] = TocEvidence( page_index=page, source="vlm", - confidence=max(0.0, min(1.0, confidence)), reason=str(item.get("reason") or ""), ) return confirmed_pages, evidence_by_page @@ -159,7 +146,6 @@ def _confirm_anchor_chunk( chunk: list[TocAnchorPage], *, model: str, - budget: Any | None, ) -> tuple[set[int], dict[int, TocEvidence], bool]: """Confirm one BOUNDARY_STEP_PAGES-sized anchor chunk. @@ -189,18 +175,11 @@ def _confirm_anchor_chunk( ) messages = cast(Any, [{"role": "user", "content": content_parts}]) - est = estimate_tokens(_CONFIRM_PROMPT) + len(chunk) * _CONFIRM_TOKENS_PER_PAGE - if budget and not budget.try_reserve("visual", est, stage=_CONFIRM_STAGE): - logger.warning( - "[extract.toc] insufficient visual budget for confirm chunk pages={}", - [a.page for a in chunk], - ) - return set(), {}, True try: client, resolved_model = get_vision_client(requested_model=model) resolved = resolved_model or model - raw, usage = client.chat_completion_with_usage( + raw, _usage = client.chat_completion_with_usage( messages=messages, model=resolved, temperature=0.1, @@ -208,20 +187,11 @@ def _confirm_anchor_chunk( response_format={"type": "json_object"}, usage_task="document_agent.toc_anchor_confirm", ) - if budget: - budget.commit( - "visual", - actual=usage.get("total_tokens", est), - est=est, - stage=_CONFIRM_STAGE, - ) confirmed_pages, evidence_by_page = _evidence_from_confirm_items( _parse_confirm_items(raw) ) return confirmed_pages, evidence_by_page, False except Exception as exc: - if budget: - budget.refund("visual", est=est, stage=_CONFIRM_STAGE) logger.warning( "[extract.toc] VLM confirm chunk failed pages={}: {}", [a.page for a in chunk], @@ -233,7 +203,6 @@ def _confirm_anchor_chunk( def _vlm_confirm_anchors( anchor_pages: list[TocAnchorPage], model: str, - budget: Any | None = None, ) -> tuple[list[TocAnchorPage], bool, list[TocEvidence]]: """Phase 1: confirm TOC starts in BOUNDARY_STEP_PAGES batches (concurrent).""" if not anchor_pages: @@ -247,12 +216,12 @@ def _vlm_confirm_anchors( len(anchor_pages), len(chunks), BOUNDARY_STEP_PAGES, - min(BOUNDARY_STEP_PAGES, len(chunks)), + min(TOC_VLM_CONCURRENCY, len(chunks)), ) - pool = GeventPool(size=min(BOUNDARY_STEP_PAGES, len(chunks))) + pool = GeventPool(size=min(TOC_VLM_CONCURRENCY, len(chunks))) jobs = [ - pool.spawn(_confirm_anchor_chunk, chunk, model=model, budget=budget) + pool.spawn(_confirm_anchor_chunk, chunk, model=model) for chunk in chunks ] pool.join() @@ -284,7 +253,6 @@ def _vlm_confirm_anchors( TocEvidence( page_index=a.page, source="vlm", - confidence=0.05, reason=( "confirm batch failed for this candidate" if confirm_failed @@ -375,6 +343,39 @@ def _render_toc_page_batch( return page_pngs +def _contiguous_toc_prefix( + page_results: list[BatchPageResult], +) -> tuple[list[int], list[dict[str, Any]]]: + """Keep TOC pages/entries from the batch start until the first non-TOC.""" + kept_pages: list[int] = [] + kept_entries: list[dict[str, Any]] = [] + for page_result in page_results: + if not page_result.is_toc: + break + kept_pages.append(int(page_result.page)) + kept_entries.extend(list(page_result.entries or [])) + return kept_pages, kept_entries + + +def _should_expand_toc_window( + *, + batch_start: int, + non_toc_pages: list[int], + kept_toc_pages: list[int], +) -> bool: + """Expand only when this batch is a full unbroken TOC window. + + Requires: + 1. ``non_toc_pages`` empty (no mid-window body / other break) + 2. last kept TOC page equals ``batch_start + BOUNDARY_STEP_PAGES - 1`` + (full step window closed on TOC; not a short end-of-doc batch) + """ + if non_toc_pages or not kept_toc_pages: + return False + full_window_end = batch_start + BOUNDARY_STEP_PAGES - 1 + return kept_toc_pages[-1] == full_window_end + + def _extract_region_for_anchor( anchor: TocAnchorPage, *, @@ -389,6 +390,10 @@ def _extract_region_for_anchor( Rounds within a start stay serial (continuation context). Different starts run concurrently for VLM, but page renders share ``render_lock``. + + Each VLM batch is truncated to the contiguous TOC prefix before the first + non-TOC page. Expand only when that prefix fills a full + ``BOUNDARY_STEP_PAGES`` window with ``non_toc_pages == []``. """ from app.services.document_agent.tools.vlm_toc_extractor import ( vlm_extract_toc_batch, @@ -442,9 +447,20 @@ def _extract_region_for_anchor( previous_entries=region_entries if region_entries else None, ) batch_meta.append(batch_result.meta) - region_entries.extend(batch_result.all_entries) - region_toc_pages.extend(batch_result.toc_pages) - region_scan_end = batch_end + + kept_toc_pages, kept_entries = _contiguous_toc_prefix( + batch_result.page_results + ) + region_entries.extend(kept_entries) + region_toc_pages.extend(kept_toc_pages) + if kept_toc_pages: + region_scan_end = kept_toc_pages[-1] + + should_expand = _should_expand_toc_window( + batch_start=batch_start, + non_toc_pages=list(batch_result.non_toc_pages), + kept_toc_pages=kept_toc_pages, + ) batch_trace.append( { "anchor": anchor_page, @@ -452,24 +468,27 @@ def _extract_region_for_anchor( "batch_pages": batch_pages, "toc_pages": batch_result.toc_pages, "non_toc_pages": batch_result.non_toc_pages, - "entries_found": len(batch_result.all_entries), + "kept_toc_pages": kept_toc_pages, + "entries_found": len(kept_entries), + "expanded": should_expand, } ) - last_page_is_toc = ( - batch_result.page_results - and batch_result.page_results[-1].is_toc - ) - if not last_page_is_toc: + if not should_expand: logger.info( - "[extract.toc] boundary found: last page {} is not TOC", + "[extract.toc] boundary found: kept={} non_toc={} " + "batch={}-{}", + kept_toc_pages, + batch_result.non_toc_pages, + batch_start, batch_end, ) break if batch_end >= page_count: break logger.info( - "[extract.toc] last page {} still TOC, expanding window", + "[extract.toc] full TOC window {}-{}, expanding", + batch_start, batch_end, ) @@ -521,7 +540,7 @@ def _extract_regions_for_confirmed_anchors( from gevent.lock import Semaphore from gevent.pool import Pool as GeventPool - pool_size = min(BOUNDARY_STEP_PAGES, len(confirmed)) + pool_size = min(TOC_VLM_CONCURRENCY, len(confirmed)) render_lock = Semaphore(1) logger.info( "[extract.toc] Phase 2 extract: {} confirmed starts, " @@ -624,9 +643,7 @@ def extract_toc_with_boundaries( os.makedirs(output_dir, exist_ok=True) # -- Phase 1: VLM confirm anchors (batched + concurrent) ------------------- - confirmed, confirm_failed, confirm_evidence = _vlm_confirm_anchors( - anchors, model, budget=ctx.budget - ) + confirmed, confirm_failed, confirm_evidence = _vlm_confirm_anchors(anchors, model) if confirm_failed: warnings.append("vlm_anchor_confirmation_failed") debug_info["phase1_confirmed"] = [a.page for a in confirmed] @@ -730,13 +747,6 @@ def extract_toc_with_boundaries( failure_kind="none", ) ctx.blackboard.toc_hierarchies = toc_hierarchies if toc_hierarchies else None - ctx.blackboard.global_signals["vlm_toc_entries"] = { - "model": model, - "toc_pages": all_toc_pages_sorted, - "total_entries": len(all_entries), - "entries": all_entries, - "batch_meta": batch_meta, - } # Build toc_ranges from confirmed TOC pages for summary toc_ranges_out: list[list[int]] = [] diff --git a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py index 14a8e213f..cd161a5c7 100644 --- a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py +++ b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py @@ -17,8 +17,7 @@ from loguru import logger # CJK and English TOC keywords used for first-pass anchor detection. -TOC_KEYWORDS = {"目录", "目次", "contents", "tableofcontents"} -TOC_CROSS_LINE_WINDOW = 6 +TOC_KEYWORDS = frozenset({"目录", "目次", "contents", "tableofcontents"}) # If a TOC keyword fingerprint appears on more than this fraction of total # pages, it is treated as a recurring navigation element (header/footer link) @@ -29,6 +28,10 @@ # document never has more than ~30 TOC start pages. MAX_ANCHOR_CANDIDATES = 30 +# Upper bound on consecutive lines joined when repairing a keyword split by +# newlines (e.g. 目\\n录). Derived from the longest keyword character length. +_MAX_KEYWORD_SPLIT_LINES = max(len(keyword) for keyword in TOC_KEYWORDS) + def _normalize_for_toc(text: str) -> str: """Collapse whitespace for keyword matching.""" @@ -39,62 +42,49 @@ def _meaningful_text_lines(text: str) -> list[str]: return [" ".join(line.split()) for line in text.splitlines() if line.split()] -def _match_toc_keyword(normalized_text: str) -> str | None: - for keyword in sorted(TOC_KEYWORDS, key=len, reverse=True): - if keyword in normalized_text: - return keyword - return None +def _merge_keyword_split_lines( + lines: list[str], +) -> list[tuple[str, int, int]]: + """Merge consecutive lines that together exactly equal one TOC keyword. + Only repairs newlines inside known keywords (e.g. 目+录, Table of+Contents). + Does not join arbitrary page text windows. + """ + merged: list[tuple[str, int, int]] = [] + index = 0 + while index < len(lines): + joined: tuple[str, int, int] | None = None + upper = min(_MAX_KEYWORD_SPLIT_LINES, len(lines) - index) + for part_count in range(upper, 1, -1): + parts = [lines[index + offset].strip() for offset in range(part_count)] + if _normalize_for_toc("".join(parts)) not in TOC_KEYWORDS: + continue + joined = ("".join(parts), index, index + part_count - 1) + break + if joined is not None: + merged.append(joined) + index = joined[2] + 1 + continue + merged.append((lines[index], index, index)) + index += 1 + return merged -def _find_toc_text_matches( - lines: list[str], - *, - cross_line_window: int, -) -> list[dict[str, Any]]: + +def _find_toc_text_matches(lines: list[str]) -> list[dict[str, Any]]: + """Match TOC keywords only as whole lines after keyword-split repair.""" matches: list[dict[str, Any]] = [] - direct_hit_lines: set[int] = set() - for line_idx, raw_line in enumerate(lines): - keyword = _match_toc_keyword(_normalize_for_toc(raw_line)) - if keyword is None: + for raw_line, start_idx, end_idx in _merge_keyword_split_lines(lines): + keyword = _normalize_for_toc(raw_line) + if keyword not in TOC_KEYWORDS: continue - direct_hit_lines.add(line_idx) matches.append( { "raw_line": raw_line.strip(), - "line_index": line_idx, - "line_end_index": line_idx, + "line_index": start_idx, + "line_end_index": end_idx, "match_kind": f"keyword:{keyword}", } ) - - if matches: - return matches - - # TODO: Add second-layer table-of-contents shape scoring if first-pass - # keywords produce too many candidates. Keep this layer as pure anchor - # discovery; VLM confirmation remains the semantic judge. - window = max(cross_line_window, 1) - for start_idx in range(len(lines)): - joined_parts: list[str] = [] - end_limit = min(len(lines), start_idx + window) - for end_idx in range(start_idx, end_limit): - if end_idx in direct_hit_lines: - continue - joined_parts.append(lines[end_idx].strip()) - if end_idx == start_idx: - continue - keyword = _match_toc_keyword(_normalize_for_toc("".join(joined_parts))) - if keyword is None: - continue - matches.append( - { - "raw_line": " / ".join(joined_parts), - "line_index": start_idx, - "line_end_index": end_idx, - "match_kind": f"cross_line:{keyword}", - } - ) - return matches return matches @@ -102,16 +92,12 @@ def _scan_toc_from_page_texts( page_texts: dict[int, str], *, page_count: int, - cross_line_window: int, ) -> list[dict[str, Any]]: matches: list[dict[str, Any]] = [] for page_num in range(1, page_count + 1): text = page_texts.get(page_num, "") lines = _meaningful_text_lines(text) - for match in _find_toc_text_matches( - lines, - cross_line_window=cross_line_window, - ): + for match in _find_toc_text_matches(lines): matches.append({"page": page_num, **match}) return matches @@ -206,7 +192,7 @@ def _filter_recurring_elements( @register_tool( name="find.toc_anchor_pages", description=( - "Scan full PDF page text for TOC keywords, filter recurring " + "Scan full PDF page text for whole-line TOC keywords, filter recurring " "navigation elements, then render candidate PNGs for VLM confirmation." ), preconditions=(has_page_labels, has_page_full_text), @@ -215,13 +201,9 @@ def find_toc_anchor_pages(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult start = time.monotonic() total_pages = ctx.blackboard.page_count - cross_line_window = int( - ctx.settings.get("toc_cross_line_window", TOC_CROSS_LINE_WINDOW) - ) keyword_matches = _scan_toc_from_page_texts( ctx.blackboard.page_full_text_cache, page_count=total_pages, - cross_line_window=cross_line_window, ) raw_hit_pages = {int(match["page"]) for match in keyword_matches} diff --git a/apps/worker/app/services/document_agent/tools/inspect_pages.py b/apps/worker/app/services/document_agent/tools/inspect_pages.py index 974a3b609..e04cd8152 100644 --- a/apps/worker/app/services/document_agent/tools/inspect_pages.py +++ b/apps/worker/app/services/document_agent/tools/inspect_pages.py @@ -1,4 +1,4 @@ -"""Generic inspect.pages tool: open physical pages, render, answer a question.""" +"""Generic inspect.pages tool: render physical pages to PNG and VLM-answer a question.""" from __future__ import annotations @@ -11,16 +11,48 @@ from loguru import logger from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.registry import has_page_features, register_tool _DEFAULT_PAGE_CAP = 5 +@register_tool( + name="inspect.pages", + description=( + "Render one or more physical PDF pages to PNG and answer a question via VLM. " + "Optional answer_keys request extra JSON fields besides answer." + ), + parameters={ + "type": "object", + "properties": { + "pages": { + "type": "array", + "items": {"type": "integer"}, + "description": "1-based physical pages to inspect", + }, + "question": { + "type": "string", + "description": "Question about the rendered page image(s)", + }, + "answer_keys": { + "type": "object", + "additionalProperties": {"type": "string"}, + "description": "Optional extra JSON keys → type/description hints", + }, + "page_cap": { + "type": "integer", + "description": "Max pages per call (batch size cap)", + }, + }, + "required": ["pages", "question"], + }, + preconditions=(has_page_features,), +) def inspect_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: """Open one or more physical PDF pages, render them, and answer ``question``. - Hard limits are token/loop budgets (via ``BudgetTracker``), not a total page - counter. ``inspect_page_cap`` only caps pages **per call** (batch size). + ``inspect_page_cap`` only caps pages per call (batch size). """ start = time.monotonic() raw_pages = args.get("pages") or [] @@ -47,7 +79,9 @@ def inspect_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: continue if 1 <= page <= page_count and page not in pages: pages.append(page) - page_cap = int(ctx.settings.get("inspect_page_cap") or _DEFAULT_PAGE_CAP) + page_cap = int( + args.get("page_cap") or ctx.settings.get("inspect_page_cap") or _DEFAULT_PAGE_CAP + ) pages = pages[: max(page_cap, 1)] if not pages: return ToolResult( @@ -82,21 +116,18 @@ def inspect_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) - # Token budget (optional stage, e.g. calibration). No total page-count ledger. - stage = args.get("visual_stage") or ctx.settings.get("inspect_visual_stage") - stage_name = str(stage).strip() if stage else None - est = 800 * len(rendered) + 800 - if stage_name and ctx.budget is not None: - if not ctx.budget.try_reserve("visual", est, stage=stage_name): - return ToolResult( - status="error", - error="calibration visual budget exhausted", - latency_ms=int((time.monotonic() - start) * 1000), - ) - + raw_answer_keys = args.get("answer_keys") + answer_keys = ( + {str(key): str(desc) for key, desc in raw_answer_keys.items()} + if isinstance(raw_answer_keys, dict) and raw_answer_keys + else {} + ) + schema_fields = ", ".join( + ['"answer": string'] + [f'"{key}": {desc}' for key, desc in answer_keys.items()] + ) prompt = ( "Answer the question about the provided PDF page image(s). " - 'Return strict JSON object with keys: {"answer": string}. ' + f"Return strict JSON object with keys: {{{schema_fields}}}. " "Include the word json in your reasoning.\n\n" f"Pages: {pages}\nQuestion: {question}\n" ) @@ -127,16 +158,7 @@ def inspect_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: ) payload = json.loads(raw) if raw else {} tokens_used = int((usage or {}).get("total_tokens") or 0) - if stage_name and ctx.budget is not None: - ctx.budget.commit( - "visual", - actual=tokens_used or est, - est=est, - stage=stage_name, - ) except Exception as exc: - if stage_name and ctx.budget is not None: - ctx.budget.refund("visual", est=est, stage=stage_name) logger.warning("[inspect.pages] VLM failed: {}", exc) return ToolResult( status="error", @@ -144,13 +166,17 @@ def inspect_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) + fields = {key: payload.get(key) for key in answer_keys} + result_payload: dict[str, Any] = { + "pages": pages, + "question": question, + "answer": payload.get("answer"), + } + if fields: + result_payload["fields"] = fields return ToolResult( status="ok", - payload={ - "pages": pages, - "question": question, - "answer": payload.get("answer"), - }, + payload=result_payload, latency_ms=int((time.monotonic() - start) * 1000), tokens_used=tokens_used, output_summary={"pages": pages, "answer": payload.get("answer")}, diff --git a/apps/worker/app/services/document_agent/tools/judge_toc_source.py b/apps/worker/app/services/document_agent/tools/judge_toc_source.py new file mode 100644 index 000000000..8f053ac27 --- /dev/null +++ b/apps/worker/app/services/document_agent/tools/judge_toc_source.py @@ -0,0 +1,179 @@ +"""judge.toc_source: pick between the PDF outline tree and printed TOC page text. + +Text-only comparison over ``page_full_text_cache``. Coverage decides; +granularity only breaks ties. +""" + +from __future__ import annotations + +import json +import time +from collections import Counter +from typing import Any, cast + +from loguru import logger + +from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.registry import has_page_full_text, register_tool + +OUTLINE_CHOICE = "outline" +PRINTED_TOC_CHOICE = "printed_toc" + +_INSTRUCTIONS = ( + "Two candidate sources describe the section structure of the same file.\n" + "Source 'outline' is the bookmark tree as level-prefixed lines.\n" + "Source 'printed_toc' is table-of-contents text extracted from places " + "across this document; it is TOC content only, not body text.\n" + "Choose which source covers more of the document.\n" + "Rank by coverage first: how many major sections/chapters a source lists " + "across the whole document.\n" + "Example: if outline lists only a few chapters in depth, while printed_toc " + "lists all chapters, choose printed_toc, since the printed_toc covers more content.\n" + + "Only when coverage is comparable, prefer the finer-grained source " + "(more depth levels and more entries).\n" + 'Return a strict json object with keys {"choice": "outline" or ' + '"printed_toc", "reason": string}. Include the word json.' +) + + +def merge_printed_toc_texts(page_texts: list[str]) -> str: + """Drop lines shared by two or more TOC pages; join remaining lines in order.""" + pages_lines: list[list[str]] = [] + for text in page_texts: + lines = [line.strip() for line in str(text).splitlines() if line.strip()] + if lines: + pages_lines.append(lines) + if not pages_lines: + return "" + if len(pages_lines) == 1: + return "\n".join(pages_lines[0]) + + presence: Counter[str] = Counter() + for lines in pages_lines: + for line in set(lines): + presence[line] += 1 + boilerplate = {line for line, count in presence.items() if count >= 2} + + kept: list[str] = [] + for lines in pages_lines: + for line in lines: + if line not in boilerplate: + kept.append(line) + return "\n".join(kept) + + +@register_tool( + name="judge.toc_source", + description=( + "Compare a PDF outline tree against printed TOC page text and choose " + "the source with broader coverage (finer granularity breaks ties)." + ), + parameters={ + "type": "object", + "properties": { + "outline_digest": { + "type": "string", + "description": "Level-prefixed outline tree digest", + }, + "toc_pages": { + "type": "array", + "items": {"type": "integer"}, + "description": "1-based physical pages holding the printed TOC", + }, + }, + "required": ["outline_digest", "toc_pages"], + }, + preconditions=(has_page_full_text,), +) +def judge_toc_source(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + start = time.monotonic() + outline_digest = str(args.get("outline_digest") or "").strip() + raw_pages = args.get("toc_pages") or [] + if not outline_digest: + return ToolResult( + status="error", + error="judge.toc_source requires outline_digest", + latency_ms=int((time.monotonic() - start) * 1000), + ) + if not isinstance(raw_pages, list) or not raw_pages: + return ToolResult( + status="error", + error="judge.toc_source requires toc_pages[]", + latency_ms=int((time.monotonic() - start) * 1000), + ) + + pages: list[int] = [] + for item in raw_pages: + try: + page = int(item) + except (TypeError, ValueError): + continue + if page not in pages: + pages.append(page) + if not pages: + return ToolResult( + status="error", + error="judge.toc_source has no valid toc_pages", + latency_ms=int((time.monotonic() - start) * 1000), + ) + + cache = dict(ctx.blackboard.page_full_text_cache) + printed_toc = merge_printed_toc_texts([cache.get(page, "") for page in pages]) + if not printed_toc.strip(): + return ToolResult( + status="error", + error="judge.toc_source printed_toc text empty", + latency_ms=int((time.monotonic() - start) * 1000), + ) + + prompt = ( + f"{_INSTRUCTIONS}\n\n" + f"Source outline:\n{outline_digest}\n\n" + f"Source printed_toc:\n{printed_toc}\n" + ) + + try: + from shared.services.ai.llm_overrides import get_text_client + + client, model = get_text_client() + raw, usage = client.chat_completion_with_usage( + messages=cast(Any, [{"role": "user", "content": prompt}]), + model=model, + temperature=0.0, + max_tokens=400, + response_format={"type": "json_object"}, + usage_task="document_agent.judge_toc_source", + ) + payload = json.loads(raw) if raw else {} + except Exception as exc: + logger.warning("[judge.toc_source] LLM failed: {}", exc) + return ToolResult( + status="error", + error=f"llm failed: {exc}", + latency_ms=int((time.monotonic() - start) * 1000), + ) + + choice = str(payload.get("choice") or "").strip().lower() + if choice not in {OUTLINE_CHOICE, PRINTED_TOC_CHOICE}: + return ToolResult( + status="error", + error=f"judge.toc_source returned unknown choice: {choice!r}", + latency_ms=int((time.monotonic() - start) * 1000), + ) + + reason = str(payload.get("reason") or "") + tokens_used = int((usage or {}).get("total_tokens") or 0) + logger.info( + "[judge.toc_source] choice={} toc_pages={} reason={}", + choice, + pages, + reason, + ) + return ToolResult( + status="ok", + payload={"choice": choice, "reason": reason, "toc_pages": pages}, + latency_ms=int((time.monotonic() - start) * 1000), + tokens_used=tokens_used, + output_summary={"choice": choice, "toc_page_count": len(pages)}, + ) diff --git a/apps/worker/app/services/document_agent/tools/persist_anatomy_map.py b/apps/worker/app/services/document_agent/tools/persist_anatomy_map.py index b7683f76b..48d16c14e 100644 --- a/apps/worker/app/services/document_agent/tools/persist_anatomy_map.py +++ b/apps/worker/app/services/document_agent/tools/persist_anatomy_map.py @@ -35,19 +35,14 @@ def build_anatomy_map(ctx: ToolContext) -> PageAnatomyMap: page_features=ctx.blackboard.page_features, page_labels=ctx.blackboard.page_labels, toc_result=ctx.blackboard.toc_result, - h1_result=ctx.blackboard.h1_result, shard_plan=ctx.blackboard.shard_plan, document_profile=ctx.blackboard.document_profile, toc_hierarchies=ctx.blackboard.toc_hierarchies, - toc_page_offset=ctx.blackboard.toc_page_offset, skeleton_anchor=ctx.blackboard.skeleton_anchor, skeleton_nodes=ctx.blackboard.skeleton_nodes, pending_skeleton_anchors=list(ctx.blackboard.pending_skeleton_anchors), global_signals=ctx.blackboard.global_signals, - trace_summary={ - "budget": ctx.budget.snapshot(), - "validation": ctx.blackboard.validation_report, - }, + trace_summary={"validation": ctx.blackboard.validation_report}, ) diff --git a/apps/worker/app/services/document_agent/tools/probe_links.py b/apps/worker/app/services/document_agent/tools/probe_links.py new file mode 100644 index 000000000..40b62b93b --- /dev/null +++ b/apps/worker/app/services/document_agent/tools/probe_links.py @@ -0,0 +1,218 @@ +"""probe.links: collect internal page hyperlinks with noise markers. + +Reads ``page.get_links()`` only. Does not attach or enrich TOC hierarchies. + +Page-number convention (all 1-based after normalize): + - ``get_links()`` dest ``page``: ``int`` is 0-based (+1); digit ``str`` is + already 1-based (unresolved URI parse). +""" + +from __future__ import annotations + +import re +import time +from collections import Counter +from dataclasses import dataclass +from typing import Any + +from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.registry import has_page_features, register_tool + +_PURE_PAGE_ANCHOR = re.compile( + r"^(" + r"[\d\s.\-–—/]+|" + r"第?\s*\d+\s*[页頁]|" + r"p\.?\s*\d+" + r")$", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class PageLink: + source_page: int # 1-based + dest_physical_page: int # 1-based + anchor_text: str + kind: int | None = None + from_y0: float | None = None + page_height: float | None = None + + +def _anchor_text_for_rect(page: Any, rect: Any) -> str: + import fitz + + words = page.get_text("words") or [] + hit: list[tuple[float, float, str]] = [] + target = fitz.Rect(rect) + # Slightly expand so thin link boxes still catch title glyphs. + target = target + (-2, -2, 2, 2) + for word in words: + x0, y0, x1, y1, text = word[:5] + if not str(text).strip(): + continue + if fitz.Rect(x0, y0, x1, y1).intersects(target): + hit.append((float(y0), float(x0), str(text))) + hit.sort() + return " ".join(part for _, _, part in hit).strip() + + +def _link_dest_physical_page(raw_page: Any) -> int: + """Normalize ``page.get_links()`` destination to 1-based physical page. + + PyMuPDF exposes two shapes for the same field: + - ``int``: resolved name-tree / GOTO path → 0-based → add 1 + - digit ``str``: unresolved URI parse (``uri_to_dict``) → already 1-based + """ + if isinstance(raw_page, int): + return raw_page + 1 + return int(raw_page) + + +def collect_page_links(pdf_path: str, pages: list[int]) -> list[PageLink]: + """Collect internal page hyperlinks on the given pages with nearby anchor text.""" + import fitz + + if not pages: + return [] + + out: list[PageLink] = [] + doc = fitz.open(pdf_path) + try: + for source_page in pages: + if source_page < 1 or source_page > doc.page_count: + continue + page = doc[source_page - 1] + page_height = float(page.rect.height) if page.rect is not None else None + for link in page.get_links() or []: + dest_raw = link.get("page") + if dest_raw is None: + continue + try: + dest_physical = _link_dest_physical_page(dest_raw) + except (TypeError, ValueError): + continue + if dest_physical < 1 or dest_physical > doc.page_count: + continue + rect = link.get("from") + if rect is None: + continue + anchor = _anchor_text_for_rect(page, rect) + if not anchor: + continue + kind = link.get("kind") + from_y0 = float(rect.y0) if hasattr(rect, "y0") else float(rect[1]) + out.append( + PageLink( + source_page=source_page, + dest_physical_page=dest_physical, + anchor_text=anchor, + kind=int(kind) if kind is not None else None, + from_y0=from_y0, + page_height=page_height, + ) + ) + finally: + doc.close() + return out + + +def _is_pure_page_anchor(text: str) -> bool: + cleaned = " ".join(str(text or "").split()) + if not cleaned: + return True + return bool(_PURE_PAGE_ANCHOR.match(cleaned)) + + +def _is_header_zone(link: PageLink) -> bool: + if link.from_y0 is None or link.page_height is None or link.page_height <= 0: + return False + return float(link.from_y0) <= float(link.page_height) * 0.12 + + +def annotate_link_noise(links: list[PageLink]) -> list[dict[str, Any]]: + """Mark pure page-number anchors, header-zone links, and repeated destinations.""" + dest_counts = Counter(link.dest_physical_page for link in links) + out: list[dict[str, Any]] = [] + for link in links: + noise: list[str] = [] + if _is_pure_page_anchor(link.anchor_text): + noise.append("pure_page_number") + if _is_header_zone(link): + noise.append("header_zone") + if dest_counts[link.dest_physical_page] >= 3: + noise.append("repeated_dest") + out.append( + { + "source_page": link.source_page, + "dest_physical_page": link.dest_physical_page, + "anchor_text": link.anchor_text, + "kind": link.kind, + "noise": noise, + "is_noise": bool(noise), + } + ) + return out + + +@register_tool( + name="probe.links", + description=( + "Collect internal PDF page hyperlinks on the given pages. " + "Returns anchor text, source page, destination physical page, kind, and noise flags." + ), + parameters={ + "type": "object", + "properties": { + "pages": { + "type": "array", + "items": {"type": "integer"}, + "description": "1-based physical pages to scan for links", + }, + }, + "required": ["pages"], + }, + preconditions=(has_page_features,), +) +def probe_links(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + start = time.monotonic() + raw_pages = args.get("pages") + if not isinstance(raw_pages, list) or not raw_pages: + return ToolResult( + status="error", + error="probe.links requires pages", + latency_ms=int((time.monotonic() - start) * 1000), + ) + pages: list[int] = [] + for item in raw_pages: + try: + page = int(item) + except (TypeError, ValueError): + continue + if page not in pages: + pages.append(page) + if not pages: + return ToolResult( + status="error", + error="probe.links requires valid pages", + latency_ms=int((time.monotonic() - start) * 1000), + ) + + links = collect_page_links(ctx.pdf_path, pages) + annotated = annotate_link_noise(links) + noise_count = sum(1 for item in annotated if item["is_noise"]) + return ToolResult( + status="ok", + payload={ + "source": "pdf_links", + "pages": pages, + "links": annotated, + "link_count": len(annotated), + "noise_count": noise_count, + }, + latency_ms=int((time.monotonic() - start) * 1000), + output_summary={ + "pages": pages, + "link_count": len(annotated), + "noise_count": noise_count, + }, + ) diff --git a/apps/worker/app/services/document_agent/tools/probe_outline.py b/apps/worker/app/services/document_agent/tools/probe_outline.py new file mode 100644 index 000000000..99093ca6b --- /dev/null +++ b/apps/worker/app/services/document_agent/tools/probe_outline.py @@ -0,0 +1,138 @@ +"""probe.outline: read PDF bookmarks via get_toc and build a pruned tree.""" + +from __future__ import annotations + +import time +from typing import Any + +from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.registry import has_page_features, register_tool + + +def _normalize_page(raw: Any) -> int | None: + """PyMuPDF outline page is 1-based; ``<= 0`` means no destination page.""" + try: + page = int(raw) + except (TypeError, ValueError): + return None + return page if page > 0 else None + + +def _flat_toc_to_forest(entries: list[list[Any]]) -> list[dict[str, Any]]: + """Convert flat ``[level, title, page]`` rows into a nested forest.""" + roots: list[dict[str, Any]] = [] + stack: list[dict[str, Any]] = [] + for row in entries: + if not isinstance(row, (list, tuple)) or len(row) < 3: + continue + level = int(row[0]) + title = str(row[1] or "").strip() + if not title or level < 1: + continue + node: dict[str, Any] = { + "title": title, + "level": level, + "page": _normalize_page(row[2]), + "children": [], + } + while stack and int(stack[-1]["level"]) >= level: + stack.pop() + if stack: + stack[-1]["children"].append(node) + else: + roots.append(node) + stack.append(node) + return roots + + +def prune_outline_forest(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Keep no-page parents when descendants have pages; drop no-page leaves/subtrees.""" + kept: list[dict[str, Any]] = [] + for node in nodes: + children = prune_outline_forest(list(node.get("children") or [])) + page = node.get("page") + if page is None and not children: + # No-page leaf, or entire no-page subtree after child prune. + continue + kept.append( + { + "title": node["title"], + "level": node["level"], + "page": page, + "children": children, + } + ) + return kept + + +def build_outline_forest(entries: list[list[Any]]) -> list[dict[str, Any]]: + return prune_outline_forest(_flat_toc_to_forest(entries)) + + +def _count_nodes(nodes: list[dict[str, Any]]) -> int: + total = 0 + for node in nodes: + total += 1 + _count_nodes(list(node.get("children") or [])) + return total + + +@register_tool( + name="probe.outline", + description=( + "Read PDF bookmark outline via get_toc(simple=True) and return a pruned tree. " + "No-page parents are kept when children have pages; no-page leaves/subtrees are dropped." + ), + parameters={ + "type": "object", + "properties": {}, + "required": [], + }, + preconditions=(has_page_features,), +) +def probe_outline(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + del args # no parameters + start = time.monotonic() + import fitz + + try: + doc = fitz.open(ctx.pdf_path) + try: + raw = doc.get_toc(simple=True) or [] + page_count = int(doc.page_count) + finally: + doc.close() + except Exception as exc: + ctx.blackboard.pdf_outline_roots = [] + return ToolResult( + status="ok", + payload={ + "source": "pdf_outline", + "page_count": 0, + "raw_entry_count": 0, + "node_count": 0, + "roots": [], + "error": f"probe.outline failed: {exc}", + }, + latency_ms=int((time.monotonic() - start) * 1000), + output_summary={"raw_entry_count": 0, "node_count": 0, "root_count": 0}, + ) + + entries = [list(row) for row in raw if isinstance(row, (list, tuple))] + forest = build_outline_forest(entries) + ctx.blackboard.pdf_outline_roots = forest + return ToolResult( + status="ok", + payload={ + "source": "pdf_outline", + "page_count": page_count, + "raw_entry_count": len(entries), + "node_count": _count_nodes(forest), + "roots": forest, + }, + latency_ms=int((time.monotonic() - start) * 1000), + output_summary={ + "raw_entry_count": len(entries), + "node_count": _count_nodes(forest), + "root_count": len(forest), + }, + ) diff --git a/apps/worker/app/services/document_agent/tools/probe_page_features.py b/apps/worker/app/services/document_agent/tools/probe_page_features.py index 5c1e83553..7369da44e 100644 --- a/apps/worker/app/services/document_agent/tools/probe_page_features.py +++ b/apps/worker/app/services/document_agent/tools/probe_page_features.py @@ -432,7 +432,6 @@ def probe_page_features(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult: ] ctx.blackboard.page_features = sorted(features, key=lambda f: f.page) ctx.blackboard.page_count = int(result.get("page_count") or len(features)) - ctx.blackboard.global_signals["total_pages"] = ctx.blackboard.page_count ctx.blackboard.global_signals["assets_probed"] = False logger.info("[document_agent] probed text on {} pages", ctx.blackboard.page_count) return ToolResult( diff --git a/apps/worker/app/services/document_agent/tools/propose_shard_plan.py b/apps/worker/app/services/document_agent/tools/propose_shard_plan.py index 9c65bffb5..ccbb794ce 100644 --- a/apps/worker/app/services/document_agent/tools/propose_shard_plan.py +++ b/apps/worker/app/services/document_agent/tools/propose_shard_plan.py @@ -21,6 +21,8 @@ ) from app.services.document_agent.structure.hierarchy_locator import ( ResolvedHierarchyRange, + TitleMatch, + TitleNode, resolve_hierarchy_page_ranges, ) from app.services.document_agent.structure.toc_anchoring import ( @@ -31,151 +33,159 @@ from app.services.document_agent.validators import single_shard_plan, validate_shard_plan -def split_toc_for_shard( - toc_hierarchies: list[dict[str, Any]] | None, - shard_page_start: int, - shard_page_end: int, +@dataclass(frozen=True) +class _AnchoredTocEntry: + heading: str + level: int + physical_page: int + path_titles: tuple[str, ...] + + +def _walk_anchored_entries( + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], *, - offset_override: int | None = None, -) -> list[dict[str, Any]] | None: - """Build per-shard toc_hierarchies filtered to the shard's page range. + prefix: tuple[str, ...] = (), +) -> list[_AnchoredTocEntry]: + """DFS title tree; keep only nodes already pinned to a physical page.""" + entries: list[_AnchoredTocEntry] = [] + for node in nodes: + path = (*prefix, node.title) + match = match_overrides.get(path) + if match is not None: + entries.append( + _AnchoredTocEntry( + heading=node.title, + level=int(node.level), + physical_page=int(match.page), + path_titles=path, + ) + ) + if node.children: + entries.extend( + _walk_anchored_entries( + list(node.children), + match_overrides, + prefix=path, + ) + ) + return entries - For continuation shards (not starting at page 1), the ancestor chain of - the first entry is prepended so downstream heading prediction has the - full structural context. - Requires calibrated ``offset_override`` for page-unit TOC regions. - """ - if not toc_hierarchies: - return None - if offset_override is None: - # Without a calibrated offset, keep non-page TOC payloads as-is and - # skip page-unit hierarchies rather than inventing arithmetic offsets. - kept = [ - hier - for hier in toc_hierarchies - if hier.get("toc_range_unit") != "page" +def _collect_calibrated_toc_entries(ctx: ToolContext) -> list[_AnchoredTocEntry]: + """Primary (post-graft) + parallel pending entries with calibrated pages.""" + entries: list[_AnchoredTocEntry] = [] + anchor_raw = ctx.blackboard.skeleton_anchor + nodes_raw = ctx.blackboard.skeleton_nodes + if isinstance(anchor_raw, dict) and isinstance(nodes_raw, list): + nodes = [ + deserialize_title_node(node) + for node in nodes_raw + if isinstance(node, dict) ] - return kept or None + if nodes: + anchor = deserialize_skeleton_anchor(anchor_raw) + entries.extend( + _walk_anchored_entries(nodes, anchor.match_overrides) + ) - result: list[dict[str, Any]] = [] - for hier in toc_hierarchies: - if hier.get("toc_range_unit") != "page": - result.append(hier) + for record in ctx.blackboard.pending_skeleton_anchors or []: + if record.get("relationship") != "parallel": continue - toc_range = hier.get("toc_range") - entries = hier.get("toc_with_level") - if not toc_range or not entries: + pending_anchor_raw = record.get("skeleton_anchor") + pending_nodes_raw = record.get("nodes") or [] + if not isinstance(pending_anchor_raw, dict) or not isinstance( + pending_nodes_raw, list + ): continue - if isinstance(entries, str): - entries = _parse_toc_with_level_entries(entries) - if not entries: + pending_nodes = [ + deserialize_title_node(node) + for node in pending_nodes_raw + if isinstance(node, dict) + ] + if not pending_nodes: continue + pending_anchor = deserialize_skeleton_anchor(pending_anchor_raw) + entries.extend( + _walk_anchored_entries(pending_nodes, pending_anchor.match_overrides) + ) + return entries - offset = offset_override - shard_entries: list[dict[str, Any]] = [] - first_idx: int | None = None - for idx, entry in enumerate(entries): - pn = entry.get("page_number") - if not isinstance(pn, int): - continue - physical = pn + offset - if shard_page_start <= physical <= shard_page_end: - if first_idx is None: - first_idx = idx - shard_entries.append(entry) +def _toc_hierarchies_for_shard( + entries: list[_AnchoredTocEntry], + *, + shard_page_start: int, + shard_page_end: int, +) -> list[dict[str, Any]] | None: + """Slice calibrated TOC entries to a shard and prepend open ancestors.""" + if not entries: + return None - if not shard_entries or first_idx is None: - continue + shard_entries: list[_AnchoredTocEntry] = [] + first_idx: int | None = None + for idx, entry in enumerate(entries): + if shard_page_start <= entry.physical_page <= shard_page_end: + if first_idx is None: + first_idx = idx + shard_entries.append(entry) + if not shard_entries or first_idx is None: + return None + + # Reopen the first entry's real TOC ancestors so the shard slice keeps its + # place in the tree. Ancestors precede the shard in this pre-order walk. + preceding_by_path = {entry.path_titles: entry for entry in entries[:first_idx]} + first_path = shard_entries[0].path_titles + ancestors = [ + {"heading": ancestor.heading, "level": ancestor.level} + for ancestor in ( + preceding_by_path.get(first_path[:depth]) + for depth in range(1, len(first_path)) + ) + if ancestor is not None + ] - # Prepend ancestor chain for continuation shards. Walk forward through - # every entry preceding the shard's first entry, maintaining a - # monotonic stack of "open" ancestors: an incoming entry closes out - # (pops) any stack entries at the same or deeper level before being - # pushed itself. A final pop against first_entry_level removes a - # trailing sibling that shares the same level as the shard's first - # entry (siblings are not ancestors). This is robust to non-monotonic - # level sequences (e.g. [L1, L2, L1, L3]), unlike a simple - # "smallest-unseen-level" scan. - first_entry_level = shard_entries[0].get("level", 1) - ancestors: list[dict[str, Any]] = [] - if first_entry_level > 1: - stack: list[dict[str, Any]] = [] - for ancestor in entries[:first_idx]: - ancestor_level = ancestor.get("level", 1) - while stack and stack[-1].get("level", 1) >= ancestor_level: - stack.pop() - stack.append(ancestor) - while stack and stack[-1].get("level", 1) >= first_entry_level: - stack.pop() - ancestors = [ - { - "heading": node.get("heading"), - "level": node.get("level", 1), - "page_number": None, - } - for node in stack - ] - - result.append({ + toc_with_level = ancestors + [ + {"heading": entry.heading, "level": entry.level} + for entry in shard_entries + ] + return [ + { "toc_range": [shard_page_start, shard_page_end], "toc_range_unit": "page", - "source": hier.get("source", "vlm_shard_split"), - "toc_with_level": ancestors + shard_entries, - }) - - return result if result else None - - -def _parse_toc_with_level_entries(markdown: str) -> list[dict[str, Any]]: - """Parse toc_with_level markdown table into list of dicts.""" - entries: list[dict[str, Any]] = [] - headers: list[str] | None = None - for raw_line in markdown.splitlines(): - line = raw_line.strip() - if not line.startswith("|") or not line.endswith("|"): - continue - cells = [cell.strip() for cell in line.strip("|").split("|")] - if not cells or all(set(cell) <= {"-", ":"} for cell in cells): - continue - if headers is None: - headers = [cell.lower() for cell in cells] - continue - row = dict(zip(headers, cells)) - level = _safe_int(row.get("level")) - heading = row.get("heading") - page_number = _safe_int(row.get("page_number")) - if heading and level: - entries.append({"heading": heading, "level": level, "page_number": page_number}) - return entries + "source": "calibrated_shard_split", + "toc_with_level": toc_with_level, + } + ] -def _safe_int(value: Any) -> int | None: - if value is None or value == "": - return None - try: - return int(value) - except (ValueError, TypeError): - return None +def _attach_shard_toc_hierarchies(ctx: ToolContext, shards: list[Shard]) -> None: + """Attach calibrated per-shard TOC slices onto the plan (in place).""" + entries = _collect_calibrated_toc_entries(ctx) + for shard in shards: + shard.toc_hierarchies = _toc_hierarchies_for_shard( + entries, + shard_page_start=shard.page_start, + shard_page_end=shard.page_end, + ) def _thresholds(ctx: ToolContext) -> tuple[int, int]: threshold = int( ctx.settings.get("shard_threshold") - or os.environ.get("PARSE_AGENT_SHARD_THRESHOLD", "200") + or os.environ.get("PARSE_PROFILE_SHARD_THRESHOLD", "200") ) max_pages = int( ctx.settings.get("max_pages_per_shard") - or os.environ.get("PARSE_AGENT_MAX_PAGES_PER_SHARD", "200") + or os.environ.get("PARSE_PROFILE_MAX_PAGES_PER_SHARD", "200") ) return threshold, max_pages -def _cuts_to_shards(cuts: list[tuple[int, str, str, float]], page_count: int) -> list[Shard]: +def _cuts_to_shards(cuts: list[tuple[int, str, str]], page_count: int) -> list[Shard]: shards: list[Shard] = [] previous = 0 - for cut_page, anchor_type, evidence, confidence in cuts: + for cut_page, anchor_type, evidence in cuts: if cut_page <= previous: continue shards.append( @@ -186,7 +196,6 @@ def _cuts_to_shards(cuts: list[tuple[int, str, str, float]], page_count: int) -> page_offset=previous, anchor_type=anchor_type, # type: ignore[arg-type] anchor_evidence=evidence, - confidence=confidence, ) ) previous = cut_page @@ -199,7 +208,6 @@ def _cuts_to_shards(cuts: list[tuple[int, str, str, float]], page_count: int) -> page_offset=previous, anchor_type="forced_max_size", anchor_evidence="final shard", - confidence=1.0, ) ) return shards @@ -243,7 +251,6 @@ def _resolve_hierarchy_forests( return [] page_count = ctx.blackboard.page_count - page_texts = dict(ctx.blackboard.page_full_text_cache or {}) toc_result = ctx.blackboard.toc_result body_pages = body_pages_excluding_toc( getattr(toc_result, "toc_pages", None) if toc_result else None, @@ -288,7 +295,6 @@ def _resolve_hierarchy_forests( primary_ranges = resolve_hierarchy_page_ranges( nodes, page_count=primary_page_count, - page_texts=page_texts, body_pages=primary_body_pages, match_overrides=anchor.match_overrides, ) @@ -316,7 +322,6 @@ def _resolve_hierarchy_forests( pending_ranges = resolve_hierarchy_page_ranges( resolve_nodes, page_count=toc_scope_end, - page_texts=page_texts, body_pages=toc_body_pages, match_overrides=pending_anchor.match_overrides, ) @@ -416,8 +421,8 @@ def _pack_range_by_blanks( end: int, max_pages: int, blank_pages: list[int], -) -> list[tuple[int, str, str, float]]: - cuts: list[tuple[int, str, str, float]] = [] +) -> list[tuple[int, str, str]]: + cuts: list[tuple[int, str, str]] = [] while end - previous > max_pages: target = previous + max_pages eligible = [ @@ -426,11 +431,11 @@ def _pack_range_by_blanks( ] if eligible: chosen = max(eligible) - cuts.append((chosen, "blank_separator", f"blank-like page at {chosen}", 0.5)) + cuts.append((chosen, "blank_separator", f"blank-like page at {chosen}")) previous = chosen else: cut_page = previous + max_pages - cuts.append((cut_page, "forced_max_size", "no separator in range", 0.2)) + cuts.append((cut_page, "forced_max_size", "no separator in range")) previous = cut_page return cuts @@ -441,8 +446,8 @@ def _hierarchy_plan( page_count: int, max_pages: int, blank_pages: list[int], -) -> list[tuple[int, str, str, float]]: - cuts: list[tuple[int, str, str, float]] = [] +) -> list[tuple[int, str, str]]: + cuts: list[tuple[int, str, str]] = [] previous = 0 for start, end in _exclusive_pieces(units): if end <= previous: @@ -458,11 +463,11 @@ def _hierarchy_plan( if range_cuts: previous = range_cuts[-1][0] if previous < end and end < page_count: - cuts.append((end, "toc_leaf_boundary", f"toc leaf at page {end + 1}", 0.85)) + cuts.append((end, "toc_leaf_boundary", f"toc leaf at page {end + 1}")) previous = end continue if end < page_count: - cuts.append((end, "toc_leaf_boundary", f"toc leaf at page {end + 1}", 0.85)) + cuts.append((end, "toc_leaf_boundary", f"toc leaf at page {end + 1}")) previous = end return cuts @@ -486,6 +491,7 @@ def propose_shard_plan(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult: threshold, max_pages = _thresholds(ctx) if page_count <= threshold: plan = single_shard_plan(page_count) + _attach_shard_toc_hierarchies(ctx, plan.shards) ctx.blackboard.shard_plan = plan return ToolResult( status="ok", @@ -516,6 +522,7 @@ def propose_shard_plan(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult: rationale = "Deterministic plan from blank-like page boundaries (no TOC)." shards = _cuts_to_shards(cuts, page_count) + _attach_shard_toc_hierarchies(ctx, shards) enabled = len(shards) > 1 if not enabled: reason = "not_needed" diff --git a/apps/worker/app/services/document_agent/tools/validate_anatomy_map.py b/apps/worker/app/services/document_agent/tools/validate_anatomy_map.py index 595dc8122..000dd6827 100644 --- a/apps/worker/app/services/document_agent/tools/validate_anatomy_map.py +++ b/apps/worker/app/services/document_agent/tools/validate_anatomy_map.py @@ -18,7 +18,7 @@ def _max_pages(ctx: ToolContext) -> int: return int( ctx.settings.get("max_pages_per_shard") - or os.environ.get("PARSE_AGENT_MAX_PAGES_PER_SHARD", "200") + or os.environ.get("PARSE_PROFILE_MAX_PAGES_PER_SHARD", "200") ) @@ -45,7 +45,6 @@ def validate_current_anatomy(ctx: ToolContext, _args: dict[str, Any]) -> ToolRes page_features=ctx.blackboard.page_features, page_labels=ctx.blackboard.page_labels, toc_result=ctx.blackboard.toc_result, - h1_result=ctx.blackboard.h1_result, shard_plan=ctx.blackboard.shard_plan, document_profile=ctx.blackboard.document_profile, global_signals=ctx.blackboard.global_signals, diff --git a/apps/worker/app/services/document_agent/tools/verdict.py b/apps/worker/app/services/document_agent/tools/verdict.py index bada1ba22..77f88f823 100644 --- a/apps/worker/app/services/document_agent/tools/verdict.py +++ b/apps/worker/app/services/document_agent/tools/verdict.py @@ -1,11 +1,11 @@ -"""Agent verdict tool.""" +"""Profile verdict tool.""" from __future__ import annotations import time from typing import Any -from app.services.document_agent.manifest import AgentVerdict, ToolContext, ToolResult +from app.services.document_agent.manifest import ProfileVerdict, ToolContext, ToolResult from app.services.document_agent.registry import has_shard_plan, register_tool @@ -27,7 +27,7 @@ def verdict(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: status = str(args.get("status") or "abort") if status not in {"success", "abort"}: status = "abort" - ctx.blackboard.verdict = AgentVerdict( + ctx.blackboard.verdict = ProfileVerdict( status=status, # type: ignore[arg-type] rationale=str(args.get("rationale") or ""), ) diff --git a/apps/worker/app/services/document_agent/trace.py b/apps/worker/app/services/document_agent/trace.py index 285525356..e813111a3 100644 --- a/apps/worker/app/services/document_agent/trace.py +++ b/apps/worker/app/services/document_agent/trace.py @@ -114,7 +114,7 @@ def persist_doc_profile(self, profile: Any | None = None) -> None: self._profile_plan_row.doc_profile = doc_profile self._db.flush() except Exception as exc: - logger.debug(f"parse agent doc profile persist failed: {exc}") + logger.debug(f"document profile persist failed: {exc}") try: if self._profile_plan_row is not None: self._db.expunge(self._profile_plan_row) @@ -140,7 +140,7 @@ def _doc_profile_for_plan(self) -> dict[str, Any] | None: "toc_pages": list(toc_result.toc_pages), "hierarchies": self._anatomy.toc_hierarchies, "evidence": [item.to_dict() for item in toc_result.evidence], - "source": "pdf_vlm" if toc_result.method != "none" else "none", + "source": toc_result.profile_source, "method": toc_result.method, "notes": toc_result.notes, "attempted": bool( @@ -206,7 +206,7 @@ def write_trace_json( encoding="utf-8", ) except Exception as exc: - logger.debug(f"parse agent trace json write failed: {exc}") + logger.debug(f"document profile trace json write failed: {exc}") def summary(self) -> dict[str, Any]: return { @@ -270,7 +270,7 @@ def flush(self, *, final_status: str, summary: dict[str, Any] | None = None) -> self._profile_plan_row.global_signals = self._anatomy.global_signals self._db.flush() except Exception as exc: - logger.debug(f"parse agent trace flush failed: {exc}") + logger.debug(f"document profile trace flush failed: {exc}") try: self._db.rollback() except Exception: diff --git a/apps/worker/app/services/document_agent/validators.py b/apps/worker/app/services/document_agent/validators.py index 7d6210021..223f529aa 100644 --- a/apps/worker/app/services/document_agent/validators.py +++ b/apps/worker/app/services/document_agent/validators.py @@ -53,7 +53,6 @@ def single_shard_plan(page_count: int) -> ShardPlan: page_offset=0, anchor_type="forced_max_size", anchor_evidence="document within shard threshold", - confidence=1.0, ) ], ) @@ -74,13 +73,6 @@ def validate_anatomy_map( errors.append("page_features do not cover every page") if label_pages != expected_pages: errors.append("page_labels do not cover every page") - toc_pages = set(anatomy.toc_result.toc_pages) - if anatomy.h1_result: - for candidate in anatomy.h1_result.h1_candidates: - if candidate.page in toc_pages: - errors.append(f"h1 candidate points to toc page {candidate.page}") - if candidate.page < 1 or candidate.page > page_count: - errors.append(f"h1 candidate page {candidate.page} out of range") shard_report = validate_shard_plan( anatomy.shard_plan, page_count=page_count, diff --git a/apps/worker/app/services/document_agent/visual.py b/apps/worker/app/services/document_agent/visual.py index 14a39578f..e09f34aa7 100644 --- a/apps/worker/app/services/document_agent/visual.py +++ b/apps/worker/app/services/document_agent/visual.py @@ -1,4 +1,4 @@ -"""Shared page rendering helpers for document-agent visual reasoning.""" +"""Shared page rendering helpers for document profile visual reasoning.""" from __future__ import annotations @@ -16,18 +16,25 @@ _DEBUG_VISUAL_DIRS = { + "coarse_profile_pages", + "calibration_verify", + "calibration_scan", + "toc_pages", + "ocr_pages", + "inspect_pages", + "profile_visuals", + # Legacy artifact dirs from older runs; keep listed so purge still cleans them. + "agent_visuals", "planner_pages", "page_locate_pages", - "toc_pages", "verify_pages", - "agent_visuals", - "ocr_pages", + "calibration_inspect", } _PAGE_MEMORY_VISUAL_DIRS = {"pages", "asset_annotate"} def visual_debug_enabled() -> bool: - return os.environ.get("DOC_AGENT_KEEP_PAGE_VISUALS", "false").strip().lower() in { + return os.environ.get("DOC_PROFILE_KEEP_PAGE_VISUALS", "false").strip().lower() in { "1", "true", "yes", @@ -82,7 +89,7 @@ def _render_pages_worker( queue.put({"ok": True, "results": results}) -def visual_output_dir(ctx: ToolContext, folder_name: str = "agent_visuals") -> str: +def visual_output_dir(ctx: ToolContext, folder_name: str = "profile_visuals") -> str: output_dir = str( Path(ctx.output_dir or os.path.expanduser("~/.knowhere/_debug_profile")) / folder_name @@ -95,7 +102,7 @@ def render_pages( ctx: ToolContext, pages: list[int], *, - folder_name: str = "agent_visuals", + folder_name: str = "profile_visuals", prefix: str = "visual", dpi: int | None = None, timeout: int = 120, @@ -107,7 +114,7 @@ def render_pages( if not bounded_pages: return [] output_dir = visual_output_dir(ctx, folder_name=folder_name) - effective_dpi = dpi or int(ctx.settings.get("agent_png_dpi", "144")) + effective_dpi = dpi or int(ctx.settings.get("profile_png_dpi", "144")) result = run_in_child_process( _render_pages_worker, ctx.pdf_path, diff --git a/apps/worker/app/services/document_parser/formats/atlas/parser.py b/apps/worker/app/services/document_parser/formats/atlas/parser.py index 90af77649..232798022 100644 --- a/apps/worker/app/services/document_parser/formats/atlas/parser.py +++ b/apps/worker/app/services/document_parser/formats/atlas/parser.py @@ -26,6 +26,10 @@ from loguru import logger from shared.core.config import settings +from shared.services.ai.token_tracking import ( + bind_token_tracker, + get_current_token_tracker_root_id, +) from shared.utils.chunk_refs import build_chunk_ref from shared.utils.text_utils import tokenize2stw_remove @@ -230,6 +234,7 @@ def parse_atlas( pd.DataFrame with ALL_DF_COLS columns """ logger.info(f"📐 Atlas pipeline: starting per-page chunking for {pdf_path}") + token_tracker_root_id = get_current_token_tracker_root_id() os.makedirs(output_dir, exist_ok=True) img_dir = os.path.join(output_dir, "images") @@ -303,8 +308,9 @@ def parse_atlas( ) def _vlm_task(page_num, img_name): - info = _vlm_extract_page_info(output_dir, img_name) - return page_num, info + with bind_token_tracker(token_tracker_root_id): + info = _vlm_extract_page_info(output_dir, img_name) + return page_num, info with ThreadPoolExecutor(max_workers=VLM_CONCURRENCY) as executor: futures = { diff --git a/apps/worker/app/services/document_parser/formats/markdown/parser.py b/apps/worker/app/services/document_parser/formats/markdown/parser.py index 32b5094b1..f863ddcb1 100755 --- a/apps/worker/app/services/document_parser/formats/markdown/parser.py +++ b/apps/worker/app/services/document_parser/formats/markdown/parser.py @@ -279,7 +279,7 @@ def parse_md( ) if toc_hierarchies is not None: - # Pre-detected TOC from upstream (e.g. DOC_AGENT VLM-based extraction). + # Pre-detected TOC from upstream (e.g. PROFILE VLM-based extraction). # Skip row-based detection entirely — TOC pages have already been # physically stripped from the PDF, so no TOC rows exist in md_lines. logger.info( diff --git a/apps/worker/app/services/document_parser/formats/pdf/parser.py b/apps/worker/app/services/document_parser/formats/pdf/parser.py index 7bfb39e16..24188dacd 100755 --- a/apps/worker/app/services/document_parser/formats/pdf/parser.py +++ b/apps/worker/app/services/document_parser/formats/pdf/parser.py @@ -26,7 +26,8 @@ def parse_pdfs( s3_key=None, job_id=None, ): - # Deprecated: prefer page_memory track for PDF processing. + # Chunk-track PDF parser (API v1 / non-page_memory). Prefer page_memory for + # v2 .pdf/.pptx (parse_track="page_memory"). base_llm_paras.update({"doc_name": filename}) # ── Atlas routing: bypass MinerU entirely ── @@ -94,11 +95,11 @@ def _parse_pdf_via_shards( """Handle PDFs via the unified shard-first hierarchy pipeline. Pipeline: - 1. DOC_AGENT → shard plan + TOC - 2. bin_pack → merged shards + 1. PROFILE → shard plan + TOC + 2. map_agent_shards → 1:1 MinerU shards 3. split_pdf (exclude TOC pages) 4. MinerU per shard (parallel) - 5. **Per-shard heading prediction** (parallel) ← NEW + 5. **Per-shard heading prediction** (parallel) 6. Merge lines_with_heading + images 7. parse_md Phase B (skip TOC detection + heading prediction) """ @@ -114,13 +115,17 @@ def _parse_pdf_via_shards( merge_shard_lines, ) from app.services.document_parser.formats.pdf.shard_splitter import ( - bin_pack_shards, + map_agent_shards, split_pdf, ) - from app.services.document_agent.tools.propose_shard_plan import split_toc_for_shard + from shared.services.ai.token_tracking import ( + bind_token_tracker, + get_current_token_tracker_root_id, + ) work_dir: str | None = None temp_shard_s3_keys: list[str] = [] + token_tracker_root_id = get_current_token_tracker_root_id() try: # 1. Reuse the entry DOC_PROFILE anatomy map (shard plan + TOC info). @@ -136,24 +141,19 @@ def _parse_pdf_via_shards( agent_shards = anatomy.shard_plan.shards - # 2. Extract TOC info from anatomy for page exclusion and heading constraint + # 2. TOC pages to exclude from physical shard PDFs. toc_pages: set[int] = set() - toc_hierarchies = anatomy.toc_hierarchies if anatomy.toc_result and anatomy.toc_result.toc_pages: toc_pages = set(anatomy.toc_result.toc_pages) logger.info( - f"📌 DOC_AGENT TOC detected: {len(toc_pages)} pages to exclude " - f"({sorted(toc_pages)}), " - f"{len(toc_hierarchies) if toc_hierarchies else 0} hierarchy regions" + f"📌 PROFILE TOC detected: {len(toc_pages)} pages to exclude " + f"({sorted(toc_pages)})" ) - # 3. Bin-pack agent shards to maximize MinerU page limit - merged_shards = bin_pack_shards( - agent_shards, - max_pages=settings.MAX_PDF_PAGE_LIMIT, - ) + # 3. Map PROFILE agent shards 1:1 onto MinerU shard jobs + merged_shards = map_agent_shards(agent_shards) logger.info( - f"📦 Bin-packed {len(agent_shards)} agent shards → " + f"📦 Mapped {len(agent_shards)} agent shards → " f"{len(merged_shards)} MinerU shards" ) for ms in merged_shards: @@ -262,14 +262,8 @@ def _predict_shard_headings( md_lines = merge_html_tables(md_lines) is_first_shard = shard_idx == 0 - shard = merged_shards[shard_idx] - shard_toc = ( - toc_hierarchies if is_first_shard - else split_toc_for_shard( - toc_hierarchies, shard.page_start, shard.page_end, - offset_override=getattr(anatomy, "toc_page_offset", None), - ) - ) + agent_shard = agent_shards[shard_idx] + shard_toc = getattr(agent_shard, "toc_hierarchies", None) lines_with_heading = eval_md_headings( md_lines, @@ -297,6 +291,13 @@ def _predict_shard_headings( heading_count=heading_count, ) + def _predict_shard_headings_with_tracking( + shard_idx: int, + shard_out_dir: str, + ) -> ShardHeadingResult: + with bind_token_tracker(token_tracker_root_id): + return _predict_shard_headings(shard_idx, shard_out_dir) + shard_heading_results: list[ShardHeadingResult | None] = [None] * len( shard_output_dirs ) @@ -306,7 +307,9 @@ def _predict_shard_headings( ): with ThreadPoolExecutor(max_workers=concurrency) as executor: futures = { - executor.submit(_predict_shard_headings, i, shard_dir): i + executor.submit( + _predict_shard_headings_with_tracking, i, shard_dir + ): i for i, shard_dir in enumerate(shard_output_dirs) if shard_dir is not None } diff --git a/apps/worker/app/services/document_parser/formats/pdf/shard_splitter.py b/apps/worker/app/services/document_parser/formats/pdf/shard_splitter.py index accc1dde6..a4f323d42 100644 --- a/apps/worker/app/services/document_parser/formats/pdf/shard_splitter.py +++ b/apps/worker/app/services/document_parser/formats/pdf/shard_splitter.py @@ -31,15 +31,14 @@ def page_offset(self) -> int: return self.page_start - 1 -def bin_pack_shards( +def map_agent_shards( agent_shards: list["Shard"], - max_pages: int, ) -> list[MergedShard]: """1:1 mapping: each agent shard becomes its own MinerU shard. - Agent shards are cut at TOC hierarchy pack boundaries by the document - agent. Merging them would cross those boundaries and degrade heading - prediction quality, so we preserve them as-is. + Agent shards are cut at TOC hierarchy pack boundaries during PROFILE. + Merging them would cross those boundaries and degrade heading prediction + quality, so we preserve them as-is. """ return [ MergedShard(idx, page_start=s.page_start, page_end=s.page_end) @@ -60,7 +59,7 @@ def split_pdf( shards: Merged shard ranges to extract. work_dir: Directory for temporary shard PDFs. exclude_pages: Optional set of 1-based page numbers to strip - (e.g. TOC pages detected by DOC_AGENT). + (e.g. TOC pages detected during PROFILE). Returns: (shard_paths, page_remap) diff --git a/apps/worker/app/services/document_parser/formats/pptx/parser.py b/apps/worker/app/services/document_parser/formats/pptx/parser.py index 9bf3a9f96..d26439c4e 100755 --- a/apps/worker/app/services/document_parser/formats/pptx/parser.py +++ b/apps/worker/app/services/document_parser/formats/pptx/parser.py @@ -1,16 +1,13 @@ # pyright: reportArgumentType=false, reportCallIssue=false import io import os -import re import time import jwt import requests -from app.services.document_parser.support.path_helpers import find_images from app.services.document_parser.conversion.legacy_converter import ( _convert_with_libreoffice, ) -from app.services.document_parser.formats.markdown.parser import parse_md from app.services.document_parser.support.parser_log_utils import truncate_log_value from app.services.document_parser.formats.pdf.rendered_transform import ( build_rendered_pdf_s3_key, @@ -18,8 +15,6 @@ parse_rendered_pdf_bytes, ) from loguru import logger -from markitdown import MarkItDown -from pptx2md import ConversionConfig, convert from shared.core.config import settings from shared.core.exceptions.domain_exceptions import ( @@ -317,12 +312,11 @@ def parse_pptx( job_id=None, ): """ - Deprecated: prefer page_memory track for PPTX processing. + Chunk-track PPTX parser (API v1). Prefer page_memory for v2 .pptx. PPTX parsing entrance, aligned with parse_pdfs / parse_docx pattern. strategy options: - - "to_md": directly extract from PPTX XML (pptx2md + MarkItDown) - "to_pdf": use LibreOffice to convert to PDF, then parse via MinerU - "to_pdf_api": use iLoveAPI to convert to PDF, then parse via MinerU (recommended) """ @@ -418,13 +412,7 @@ def parse_pptx( rendered_pdf_s3_key=rendered_pdf_s3_key, ) - elif strategy == "to_md": - return _parse_pptx_to_md( - pptx_data, filename, output_dir, base_llm_paras, relative_root - ) - - else: - raise ValueError(f"Unknown pptx strategy: {strategy}") + raise ValueError(f"Unknown pptx strategy: {strategy}") def _parse_pptx_via_api( @@ -502,54 +490,3 @@ def _parse_pptx_via_libreoffice( relative_root=relative_root, rendered_pdf_s3_key=rendered_pdf_s3_key, ) - - -def _parse_pptx_to_md(pptx_data, filename, output_dir, base_llm_paras, relative_root): - """Extract content from PPTX XML via pptx2md + MarkItDown → parse_md.""" - # pptx2md and MarkItDown require file paths - local_pptx = os.path.join(output_dir, "_pptx_tmp.pptx") - with open(local_pptx, "wb") as f: - f.write(pptx_data) - - try: - img_dir = os.path.join(output_dir, "images") - os.makedirs(img_dir, exist_ok=True) - temp_md_path = os.path.join(output_dir, "output.md") - - convert( - ConversionConfig( - pptx_path=local_pptx, output_path=temp_md_path, image_dir=img_dir - ) - ) - - md = MarkItDown(enable_plugins=False) - result = md.convert(local_pptx) - - pattern = r"^!\[.*?\]\(.*?\.(?:png|jpe?g)\)$" - md_imgs = find_images(output_dir) - lines = result.text_content.splitlines() - - ppt_md_lines = [] - image_index = 0 - for line in lines: - if image_index < len(md_imgs): - if re.match(pattern, line.strip(), re.IGNORECASE): - line = f"![image{image_index + 1}]({md_imgs[image_index]})" - image_index += 1 - ppt_md_lines.append(line) - - while image_index < len(md_imgs): - ppt_md_lines.append(f"![image{image_index + 1}]({md_imgs[image_index]})") - image_index += 1 - - parsed_df = parse_md( - output_dir, - source_type="pptx", - md_lines=ppt_md_lines, - base_llm_paras=base_llm_paras, - relative_root=relative_root, - ) - return parsed_df - finally: - if os.path.exists(local_pptx): - os.remove(local_pptx) diff --git a/apps/worker/app/services/document_parser/orchestration/format_adapters.py b/apps/worker/app/services/document_parser/orchestration/format_adapters.py index 5a26a45ba..63fb4cfb2 100644 --- a/apps/worker/app/services/document_parser/orchestration/format_adapters.py +++ b/apps/worker/app/services/document_parser/orchestration/format_adapters.py @@ -72,7 +72,7 @@ def parse(self, session: ParseSession) -> ParseOutput: @dataclass(frozen=True) class PdfParseAdapter: - # Deprecated: prefer page_memory track for PDF (parse_track="page_memory"). + # Chunk-track PDF adapter (API v1). v2 .pdf/.pptx uses parse_track="page_memory". document_format: object def parse(self, session: ParseSession) -> ParseOutput: @@ -137,11 +137,8 @@ def parse(self, session: ParseSession) -> ParseOutput: @dataclass(frozen=True) class PptxParseAdapter: - # Deprecated: prefer page_memory track for PPTX (parse_track="page_memory"). - # TODO(pptx): convert PPTX→PDF first, then reuse the standard PDF PROFILE - # path instead of a separate PPTX PROFILE; keep page_memory input schema - # stable (fields may expand later). page_memory/normalizer.py already - # converts before PROFILE. + # Chunk-track PPTX adapter (API v1). v2 .pptx uses parse_track="page_memory" + # (normalizer converts PPTX→PDF before PROFILE). document_format: object def parse(self, session: ParseSession) -> ParseOutput: diff --git a/apps/worker/app/services/document_parser/profiling/doc_profiler.py b/apps/worker/app/services/document_parser/profiling/doc_profiler.py index d041b0d5b..612d73fbe 100644 --- a/apps/worker/app/services/document_parser/profiling/doc_profiler.py +++ b/apps/worker/app/services/document_parser/profiling/doc_profiler.py @@ -44,17 +44,17 @@ def profile_document( job_id: Parse job id for profile trace artifacts output_dir: Parser output directory skip_shard_plan: When True, lightweight and structural anatomy skip - LLM/ReAct shard planning and populate a single-shard placeholder. + LLM shard planning and populate a single-shard placeholder. Used by the page-memory track, which never consumes the shard plan. Chunk-track keeps the default (False) so oversized MinerU sharding still receives a real plan. oversized_policy: Controls oversized PDF admission. ``chunk`` applies the MinerU shard gate, while ``page_memory`` lets the page-memory track continue to structural profiling. - skip_toc_anchoring: When True, TOC find/extract/link-attach still run, - but ``run_toc_anchoring`` is skipped. Used by the staged debug - path (Stage-1 TOC after Stage-0 bootstrap) so calibration stays - in Stage-2. + skip_toc_anchoring: When True, TOC find/extract still run, but + ``run_toc_anchoring`` is skipped. Used by the staged debug path + (Stage-1 TOC after Stage-0 bootstrap) so calibration stays in + Stage-2. Returns: ParserDocumentProfile @@ -122,10 +122,9 @@ def _profile_pdf_with_db( ) -> ParserDocumentProfile: profile_job_id = job_id or filename agent_output_dir = os.path.join(output_dir, "_doc_agent") if output_dir else None - # Page-memory sections are anchored on the TOC (page-based VLM TOC pipeline), - # so TOC profiling is mandatory for that track regardless of the global - # PDF_PROFILE_TOC_ENABLED flag (which only gates the optional chunk-track - # TOC profiling that can otherwise fall back to MinerU markdown headings). + # PROFILE TOC is default-on for both tracks. page_memory always forces it + # on (sections are TOC-anchored) even if PDF_PROFILE_TOC_ENABLED is used as + # an emergency kill switch for the chunk track. page_toc_enabled = ( oversized_policy == "page_memory" or settings.PDF_PROFILE_TOC_ENABLED ) @@ -136,7 +135,6 @@ def _profile_pdf_with_db( db=db, model=settings.IMAGE_MODEL, settings={ - "planner_model": settings.IMAGE_MODEL, "vlm_model": settings.IMAGE_MODEL, "toc_profile_enabled": page_toc_enabled, "skip_toc_anchoring": bool(skip_toc_anchoring), @@ -214,12 +212,11 @@ def _map_toc_profile(coordinator: ProfileCoordinator) -> ParserTocProfile: TocEvidence( page_index=item.page_index, source=item.source, - confidence=item.confidence, reason=item.reason, ) for item in toc_result.evidence ] - source = "pdf_vlm" if toc_result.method != "none" else "none" + source = toc_result.profile_source return ParserTocProfile( toc_pages=list(toc_result.toc_pages), hierarchies=coordinator.blackboard.toc_hierarchies, diff --git a/apps/worker/app/services/document_parser/profiling/profile_model.py b/apps/worker/app/services/document_parser/profiling/profile_model.py index 1d3fdde5a..d711a7c09 100644 --- a/apps/worker/app/services/document_parser/profiling/profile_model.py +++ b/apps/worker/app/services/document_parser/profiling/profile_model.py @@ -10,7 +10,6 @@ class TocEvidence: page_index: int source: str - confidence: float reason: str = "" @@ -41,7 +40,6 @@ class ParserDocumentProfile: language: str = "unknown" reasoning: str = "" toc: ParserTocProfile = field(default_factory=ParserTocProfile) - granularity: str = "page" anatomy: Any | None = None metrics: dict[str, Any] = field(default_factory=dict) page_full_text_cache: dict[int, str] = field(default_factory=dict) diff --git a/apps/worker/app/services/document_parser/structure/body_boundary.py b/apps/worker/app/services/document_parser/structure/body_boundary.py index a7ad6d3db..f0ce15e42 100644 --- a/apps/worker/app/services/document_parser/structure/body_boundary.py +++ b/apps/worker/app/services/document_parser/structure/body_boundary.py @@ -41,17 +41,37 @@ def clean_toc_title(title: str) -> str: def extract_level1_titles(toc_hierarchies: list[dict[str, Any]]) -> list[str]: - """Extract cleaned level-1 titles from TOC hierarchy payloads.""" + """Extract cleaned level-1 titles from ``toc_with_level`` payloads. + + TEXT-TRACK PDF shards attach calibrated TOC slices that only carry + ``toc_with_level`` (no ``toc_tree``). Prefer that flat list so front-matter + demotion keeps working after PROFILE skeleton reuse. + """ titles: list[str] = [] for hier in toc_hierarchies: - toc_tree = hier.get("toc_tree") or {} - for raw_title in toc_tree.keys(): - cleaned = clean_toc_title(str(raw_title)) + for entry in _iter_toc_with_level_entries(hier.get("toc_with_level")): + level = entry.get("level") + if level is None: + continue + try: + if int(level) != 1: + continue + except (TypeError, ValueError): + continue + heading = entry.get("heading") or entry.get("title") or "" + cleaned = clean_toc_title(str(heading)) if cleaned and len(cleaned) >= 2: titles.append(cleaned) return titles +def _iter_toc_with_level_entries(payload: Any) -> list[dict[str, Any]]: + """Normalize ``toc_with_level`` to a list of entry dicts.""" + if isinstance(payload, list): + return [entry for entry in payload if isinstance(entry, dict)] + return [] + + def find_first_body_boundary( lines: list[str], level1_titles: list[str], diff --git a/apps/worker/app/services/document_parser/structure/heading_llm_executor.py b/apps/worker/app/services/document_parser/structure/heading_llm_executor.py index 07fe1c6d0..cbd31baea 100644 --- a/apps/worker/app/services/document_parser/structure/heading_llm_executor.py +++ b/apps/worker/app/services/document_parser/structure/heading_llm_executor.py @@ -21,24 +21,15 @@ def compact_for_llm(df: pd.DataFrame) -> pd.DataFrame: """Collapse consecutive body rows into placeholder rows before LLM chunking. - The output DataFrame has columns [id, heading, note, reason]. - - ``note`` is used internally by ``run_merge_pre_pass`` only — it is - NOT forwarded to the main hierarchy LLM: - - ``"?"`` marks a heading candidate that is **directly adjacent** to the - previous candidate with **no placeholder between them**. This signals - to the pre-pass that the pair should be evaluated for possible merging. - - ``""`` (empty) for all other rows (normal candidates and placeholders). - + The output DataFrame has columns [id, heading, reason]. The ``level`` column is intentionally NOT forwarded to the LLM. """ if df is None or len(df) == 0: - return pd.DataFrame(columns=["id", "heading", "note", "reason"]) + return pd.DataFrame(columns=["id", "heading", "reason"]) rows: list[dict[str, Any]] = [] index = 0 row_count = len(df) - prev_was_candidate = False # True when the immediately preceding output row is a candidate while index < row_count: lvl_raw = df.iloc[index]["level"] @@ -64,28 +55,22 @@ def compact_for_llm(df: pd.DataFrame) -> pd.DataFrame: { "id": f"{start_id}-{end_id}", "heading": f"[{run_length} BODY LINES]", - "note": "", "reason": PLACEHOLDER_REASON, } ) - prev_was_candidate = False index = end_index else: row = df.iloc[index] - # Mark with '?' when directly following another candidate (no placeholder gap) - note = "?" if prev_was_candidate else "" rows.append( { "id": int(row["id"]), "heading": str(row["heading"]), - "note": note, "reason": str(row.get("reason", "") or ""), } ) - prev_was_candidate = True index += 1 - return pd.DataFrame(rows, columns=["id", "heading", "note", "reason"]) + return pd.DataFrame(rows, columns=["id", "heading", "reason"]) def split_heading_table( @@ -118,127 +103,6 @@ def split_heading_table( return sub_dfs, raw_headings -def run_merge_pre_pass( - compact_df: pd.DataFrame, - model_name: str | None = None, -) -> dict[int, str]: - """DEPRECATED — no longer wired into the pipeline. Kept for reference only. - - This focused "are these two consecutive lines one split heading?" pre-pass - proved unreliable: the LLM frequently merged real chapter headings (e.g. - ``第一章 总则``) into the preceding document title, irreversibly destroying - structure. The merge concern is now handled declaratively by Rule 2 of the - ``eval-headings`` prompt ("no body between two candidates ⇒ not same level"). - - DO NOT call this. Retained so the approach can be revisited if needed. - - --- - Focused pre-pass: decide merge/keep for consecutive heading candidate groups. - Scans ``compact_df`` (output of ``compact_for_llm``) for rows whose - ``note == "?"``, groups them with their preceding candidate, and sends all - groups in a single ``eval-merge-groups`` LLM call. Returns ``{row_id: "<"}`` - for every row the LLM decides to merge into the previous heading. - """ - # ── 1. Collect consecutive groups ── - groups: list[list[dict[str, Any]]] = [] # each element: list of {id, heading} - current_group: list[dict[str, Any]] = [] - - for _, row in compact_df.iterrows(): - if row.get("reason") == PLACEHOLDER_REASON: - # Body-text placeholder breaks any running group - if len(current_group) >= 2: - groups.append(current_group) - current_group = [] - continue - - note = str(row.get("note", "")) - if note == "?": - # Continuation of a consecutive run - current_group.append({"id": int(row["id"]), "heading": str(row["heading"])}) - else: - # Start of a new candidate — flush previous group if large enough - if len(current_group) >= 2: - groups.append(current_group) - current_group = [{"id": int(row["id"]), "heading": str(row["heading"])}] - - if len(current_group) >= 2: - groups.append(current_group) - - if not groups: - logger.info("merge pre-pass: no consecutive groups found, skipping") - return {} - - logger.info(f"merge pre-pass: {len(groups)} consecutive group(s) to evaluate") - - # ── 2. Format groups for the prompt ── - lines: list[str] = [] - for g_idx, group in enumerate(groups, start=1): - headings_str = " | ".join(f'"{item["heading"]}"' for item in group) - lines.append(f"Group {g_idx}: [{headings_str}]") - texts = "\n".join(lines) - - # ── 3. Call LLM directly (bypass df2md — texts is already formatted) ── - from shared.services.ai.prompt_service import build_prompt - from shared.services.ai.llm_overrides import get_text_client - from shared.services.ai.response_process_service import eval_response - - try: - prompt, temperature, top_p, max_tokens = build_prompt( - task="eval-merge-groups", - texts=texts, - query="", - paras={"max_tokens": min(800, len(groups) * 50 + 200)}, - ) - messages = [ - {"role": "system", "content": "you are a document structure expert"}, - {"role": "user", "content": prompt}, - ] - with stage_timer("heading.merge_pre_pass_llm", group_count=len(groups), model_name=model_name): - client, model_name = get_text_client(requested_model=model_name) - answer = client.chat_completion( - messages=messages, - model=model_name, - max_tokens=max_tokens, - temperature=temperature, - usage_task="parser.heading_merge_pre_pass", - ) - result = eval_response(answer) - except Exception as exc: - logger.warning(f"merge pre-pass LLM call failed: {exc}, skipping pre-pass") - return {} - - - # ── 4. Parse result → {id: "<"} ── - merge_ids: dict[int, str] = {} - if not isinstance(result, list): - logger.warning(f"merge pre-pass: unexpected result type {type(result)}, skipping") - return {} - - for item in result: - if not isinstance(item, dict): - continue - g_idx = item.get("group") - should_merge = item.get("merge", False) - if not should_merge: - continue - try: - g_idx = int(g_idx) - except (TypeError, ValueError): - continue - if g_idx < 1 or g_idx > len(groups): - continue - group = groups[g_idx - 1] - # Mark all rows except the first as "<" - for member in group[1:]: - merge_ids[member["id"]] = "<" - logger.debug( - f"merge pre-pass: id={member['id']} '{member['heading'][:50]}' → '<'" - ) - - logger.info(f"merge pre-pass: {len(merge_ids)} row(s) flagged for merge") - return merge_ids - - def _coerce_level(value: Any) -> int: try: return int(value) @@ -391,10 +255,6 @@ def execute_llm_heading_hierarchy( fallback["level"] = -1 return fallback.sort_values("id").reset_index(drop=True) - # NOTE: the legacy LLM "merge pre-pass" (run_merge_pre_pass) is deprecated and - # no longer wired in. The merge concern is now handled by Rule 2 of the - # eval-headings prompt. The main hierarchy LLM assigns every level directly. - level_dfs, _raw_headings = split_heading_table( preds_for_llm, threshold=prompt_limt, max_start=max_len, max_end=5 ) @@ -430,8 +290,8 @@ def execute_llm_heading_hierarchy( ) continue - # Send only [id, heading] to the main LLM — no note, no merge hints - df4llm = chunk_df.drop(columns=["reason", "level", "note"], errors="ignore").copy() + # Send only [id, heading] to the main LLM + df4llm = chunk_df.drop(columns=["reason", "level"], errors="ignore").copy() df4llm["heading"] = df4llm["heading"].apply(clean_md_text_for_llm) logger.info( diff --git a/apps/worker/app/services/document_parser/structure/layout_parser.py b/apps/worker/app/services/document_parser/structure/layout_parser.py index ea3b8bc83..7397e9d22 100755 --- a/apps/worker/app/services/document_parser/structure/layout_parser.py +++ b/apps/worker/app/services/document_parser/structure/layout_parser.py @@ -105,9 +105,14 @@ def format_toc_context_for_llm(toc_context) -> str: toc_entries = toc_item.get("toc_with_level") or [] if toc_range and len(toc_range) == 2: - formatted_blocks.append( - f"TOC {toc_idx} (source rows {toc_range[0]}-{toc_range[1]}):" - ) + if toc_item.get("toc_range_unit") == "page": + formatted_blocks.append( + f"TOC {toc_idx} (pages {toc_range[0]}-{toc_range[1]}):" + ) + else: + formatted_blocks.append( + f"TOC {toc_idx} (source rows {toc_range[0]}-{toc_range[1]}):" + ) else: formatted_blocks.append(f"TOC {toc_idx}:") @@ -298,7 +303,7 @@ def _is_candidate_id(val): def _compute_zone_boundaries(toc_hierarchies, coordinate_mode="post_removal"): - """Compute content zone boundaries for documents with multiple TOC areas. + """Compute content zones for line/element-based TOC areas. When multiple TOCs exist, they divide the document into zones. Each zone starts right after a TOC area and extends to just before the next TOC area @@ -306,10 +311,13 @@ def _compute_zone_boundaries(toc_hierarchies, coordinate_mode="post_removal"): coordinate_mode: - "post_removal": TOC ranges are in original coordinates, but heading IDs - are measured after TOC rows were removed (MD/PDF path). + are measured after TOC rows were removed (native Markdown path). - "original": heading IDs stay in original document coordinates, so zones can be computed directly from TOC boundaries (DOCX path). + PROFILE-owned PDF TOCs use page coordinates and are sliced per shard before + this stage. They must never enter this line/element-coordinate calculation. + Args: toc_hierarchies: List of toc hierarchy dicts (sorted by toc_range start) @@ -403,10 +411,10 @@ def _resolve_first_toc_boundary(toc_hierarchies=None, first_toc_ele_num=None): """Resolve the earliest available first-TOC boundary across coordinate sources. Page-based TOC boundaries (``toc_range_unit == "page"``, produced by - DOC_AGENT/VLM for PDF/PPT) are in *page numbers*, NOT line/element IDs. + PROFILE/VLM for PDF/PPT) are in *page numbers*, NOT line/element IDs. They must NOT be used for pre-TOC row removal because ``raw_preds["id"]`` are line indices that restart from 0 in each shard. For these documents - the DOC_AGENT has already handled shard splitting around TOC pages. + PROFILE has already handled shard splitting around TOC pages. """ toc_range_start = None toc_unit = None @@ -418,12 +426,9 @@ def _resolve_first_toc_boundary(toc_hierarchies=None, first_toc_ele_num=None): # Page-based coordinates cannot be compared against line/element IDs. if toc_unit == "page": - if first_toc_ele_num is not None: - # DOCX fallback: element-based boundary is safe to use. - return first_toc_ele_num logger.debug( "📌 Skipping pre-TOC removal: TOC uses page-based coordinates " - "(DOC_AGENT already handled shard boundaries)" + "(PROFILE already handled shard boundaries)" ) return None @@ -446,10 +451,27 @@ def _resolve_first_toc_boundary(toc_hierarchies=None, first_toc_ele_num=None): return resolved_start -def _first_toc_range_unit(toc_hierarchies=None) -> str | None: - if not toc_hierarchies: - return None - return toc_hierarchies[0].get("toc_range_unit") +def _uses_page_toc_coordinates(toc_hierarchies=None) -> bool: + return bool( + toc_hierarchies + and toc_hierarchies[0].get("toc_range_unit") == "page" + ) + + +def _supports_multi_toc_zones( + toc_hierarchies, + *, + doc_type: str, + smart_parse: bool, +) -> bool: + """Whether TOC ranges share coordinates with heading candidate IDs.""" + return bool( + toc_hierarchies + and len(toc_hierarchies) > 1 + and doc_type in {"md", "docx"} + and smart_parse + and not _uses_page_toc_coordinates(toc_hierarchies) + ) def pred_titles( @@ -507,7 +529,7 @@ def pred_titles( if ( first_toc_start is None and is_first_shard - and _first_toc_range_unit(toc_hierarchies) == "page" + and _uses_page_toc_coordinates(toc_hierarchies) ): level1_titles = extract_level1_titles(toc_hierarchies or []) first_toc_start = find_first_body_boundary(infos, level1_titles) @@ -539,13 +561,11 @@ def pred_titles( f"(id < {first_toc_start}) from heading prediction" ) - # 2. Zone-based prediction when multiple TOCs exist - if ( - toc_hierarchies - and len(toc_hierarchies) > 1 - and doc_type in {"md", "docx"} - and smart_parse - and _first_toc_range_unit(toc_hierarchies) != "page" + # 2. Zone-based prediction for native Markdown/DOCX coordinates only. + if _supports_multi_toc_zones( + toc_hierarchies, + doc_type=doc_type, + smart_parse=smart_parse, ): # Multiple TOCs divide the document into independent zones. # Each zone gets its own naive + LLM pipeline with zone-specific TOC context. diff --git a/apps/worker/app/services/page_memory/memory_service.py b/apps/worker/app/services/page_memory/memory_service.py index 30ca70088..abef06c4b 100644 --- a/apps/worker/app/services/page_memory/memory_service.py +++ b/apps/worker/app/services/page_memory/memory_service.py @@ -51,9 +51,6 @@ class PageMemoryInput: ) -_HierarchyScope = CoarseScope - - @dataclass(frozen=True) class _ScopeRunResult: scope_id: str @@ -69,7 +66,7 @@ def run(request: PageMemoryInput) -> tuple[str, pd.DataFrame]: Supports two granularity verdicts: - ``whole_doc`` (≤6 pages, no TOC) → single whole-document chunk - - ``page`` → per-page chunks via the full C1-C7 pipeline + - ``page`` → leaf section nodes via the full C1-C7 pipeline """ full_output_dir = _resolve_output_dir(request) page_memory_config = request.page_memory_config @@ -433,7 +430,6 @@ def _build_page_dataframe( tag_by_page=tag_map, filename=filename, verdict=verdict, - budget=None, vlm_model=vlm_model, page_assets_by_page=page_assets_by_page, node_summary_max_pages=page_memory_config.node_summary_max_pages, @@ -461,7 +457,7 @@ def _build_hierarchy_scopes( skeletons: list[Any], filename: str, page_count: int, -) -> list[_HierarchyScope]: +) -> list[CoarseScope]: return build_hierarchy_scopes( skeletons=skeletons, filename=filename, @@ -470,7 +466,7 @@ def _build_hierarchy_scopes( def _allocate_asset_pages( - scopes: list[_HierarchyScope], total_budget: int, + scopes: list[CoarseScope], total_budget: int, ) -> list[int]: """Pre-allocate asset page budget proportionally to avoid concurrency races.""" if total_budget <= 0 or not scopes: @@ -500,7 +496,7 @@ def _run_scope_with_retry( from shared.core.exceptions.domain_exceptions import LLMServiceException - scope: _HierarchyScope = kwargs["scope"] + scope: CoarseScope = kwargs["scope"] scope_index: int = kwargs["scope_index"] scope_count: int = kwargs["scope_count"] @@ -526,7 +522,7 @@ def _run_scope_with_retry( def _run_hierarchy_scope( *, - scope: _HierarchyScope, + scope: CoarseScope, scope_index: int, scope_count: int, pdf_path: str, @@ -612,7 +608,6 @@ def _run_hierarchy_scope( pages=title_rendered, tag_results=title_tags, fat_leaf_pages=fat_leaf_pages, - budget=None, vlm_model=vlm_model, scan_direction=page_memory_config.scan_direction, max_concurrent=page_memory_config.title_detection_concurrency, @@ -708,7 +703,6 @@ def _run_hierarchy_scope( tags = tag_pages( pages=rendered, plans=plans, - budget=None, vlm_model=vlm_model, max_concurrent=page_memory_config.tag_concurrency, ) @@ -737,7 +731,6 @@ def _run_hierarchy_scope( rendered_pages=asset_rendered, output_dir=output_dir, model_name=page_memory_config.asset_model, - budget=None, max_pages=asset_max_pages, confidence_threshold=page_memory_config.asset_confidence_threshold, summary_enabled=page_memory_config.asset_summary_enabled, diff --git a/apps/worker/app/services/page_memory/node_assembler.py b/apps/worker/app/services/page_memory/node_assembler.py index a68d55c23..7273a0932 100644 --- a/apps/worker/app/services/page_memory/node_assembler.py +++ b/apps/worker/app/services/page_memory/node_assembler.py @@ -250,7 +250,6 @@ def compute_node_summary( tag_by_page: dict[int, PageTagResult], image_path_by_page: dict[int, str], vlm_model: str | None, - budget: Any | None = None, node_summary_max_pages: int = _NODE_SUMMARY_MAX_PAGES_DEFAULT, ) -> tuple[str, list[str], list[dict[str, str]]]: """Settle a node's summary, keywords, and typed entities (§4.4). @@ -387,7 +386,6 @@ def build_node_rows( tag_by_page: dict[int, PageTagResult], filename: str, verdict: str, - budget: Any | None = None, vlm_model: str | None = None, page_assets_by_page: dict[int, list[PageAsset]] | None = None, node_summary_max_pages: int = _NODE_SUMMARY_MAX_PAGES_DEFAULT, diff --git a/apps/worker/app/services/page_memory/page_assets.py b/apps/worker/app/services/page_memory/page_assets.py index 05c17fe60..c027ddb1b 100644 --- a/apps/worker/app/services/page_memory/page_assets.py +++ b/apps/worker/app/services/page_memory/page_assets.py @@ -81,7 +81,6 @@ def detect_page_assets( page: PageRenderResult, source_name: str, model_name: str | None, - budget: Any | None = None, confidence_threshold: float, ) -> list[PageAsset]: """Detect asset regions on one rendered page via VLM.""" @@ -312,7 +311,6 @@ def summarize_page_asset( *, asset: PageAsset, model_name: str | None, - budget: Any | None = None, ) -> PageAsset: """Summarize a cropped asset via the unified engine (§4.3). @@ -369,7 +367,6 @@ def extract_page_assets_from_renders( rendered_pages: list[PageRenderResult], output_dir: str, model_name: str | None, - budget: Any | None = None, max_pages: int, confidence_threshold: float, summary_enabled: bool = False, diff --git a/apps/worker/app/services/page_memory/page_renderer.py b/apps/worker/app/services/page_memory/page_renderer.py index 3019cc2ee..944d7083a 100644 --- a/apps/worker/app/services/page_memory/page_renderer.py +++ b/apps/worker/app/services/page_memory/page_renderer.py @@ -102,18 +102,17 @@ def render_document_pages( timeout=timeout, ) else: - from app.services.document_agent.state import AgentBlackboard + from app.services.document_agent.state import ProfileBlackboard - blackboard = AgentBlackboard() + blackboard = ProfileBlackboard() blackboard.page_count = page_count tmp_ctx = ToolContext( pdf_path=pdf_path, job_id="page_renderer", blackboard=blackboard, - budget=None, trace=None, output_dir=output_dir, - settings={"agent_png_dpi": str(dpi)}, + settings={"profile_png_dpi": str(dpi)}, ) pngs = render_pages( tmp_ctx, diff --git a/apps/worker/app/services/page_memory/page_tagger.py b/apps/worker/app/services/page_memory/page_tagger.py index 2204ddbd7..cd2e0ac32 100644 --- a/apps/worker/app/services/page_memory/page_tagger.py +++ b/apps/worker/app/services/page_memory/page_tagger.py @@ -55,7 +55,6 @@ def tag_pages( *, pages: list[PageRenderResult], plans: list[PagePlan], - budget: Any | None = None, vlm_model: str | None = None, max_concurrent: int | None = None, ) -> list[PageTagResult]: @@ -67,8 +66,6 @@ def tag_pages( Rendered page results (from ``page_renderer``). plans: Processing plans (from ``page_plan``). - budget: - Deprecated, ignored. Kept for call-site compatibility. vlm_model: VLM model name; falls back to ``$IMAGE_MODEL``. max_concurrent: @@ -181,7 +178,6 @@ def tag_page_titles( pages: list[PageRenderResult], tag_results: list[PageTagResult], fat_leaf_pages: set[int], - budget: Any | None = None, vlm_model: str | None = None, scan_direction: str = "top_to_bottom_left_to_right", max_concurrent: int | None = None, @@ -200,8 +196,6 @@ def tag_page_titles( fat_leaf_pages: Set of page indices belonging to fat-leaf TOC sections (those with more than the configured fine-min-page threshold). - budget: - Deprecated, ignored. Kept for call-site compatibility. vlm_model: VLM model name; falls back to ``$IMAGE_MODEL``. scan_direction: diff --git a/apps/worker/app/services/page_memory/skeleton_extractor.py b/apps/worker/app/services/page_memory/skeleton_extractor.py index 4b382e738..f7faa8420 100644 --- a/apps/worker/app/services/page_memory/skeleton_extractor.py +++ b/apps/worker/app/services/page_memory/skeleton_extractor.py @@ -129,16 +129,16 @@ def extract_section_skeletons( match_overrides = skeleton_anchor.match_overrides null_page_report = skeleton_anchor.null_page_report - if skeleton_anchor.locate_agent == "offset_guided_bulk": + if skeleton_anchor.locate_method == "offset_guided_bulk": locate_summary: dict[str, Any] = { - "agent": "offset_guided_bulk", + "locate_method": "offset_guided_bulk", "offset": skeleton_anchor.offset, "bulk_count": skeleton_anchor.bulk_count, "pruned_out_of_scope": skeleton_anchor.pruned_count, } else: locate_summary = { - "agent": "offset_only", + "locate_method": "offset_only", "offset": skeleton_anchor.offset, "reason": "offset_guided_anchoring_skipped_or_empty", "pruned_out_of_scope": skeleton_anchor.pruned_count, @@ -158,7 +158,6 @@ def extract_section_skeletons( ranges = resolve_hierarchy_page_ranges( resolve_nodes, page_count=primary_page_count, - page_texts=page_texts, body_pages=primary_body_pages, match_overrides=match_overrides, ) @@ -327,7 +326,7 @@ def _resolve_pending_tocs( match_overrides = skeleton_anchor.match_overrides null_page_report = skeleton_anchor.null_page_report locate_summary: dict[str, Any] = { - "agent": skeleton_anchor.locate_agent, + "locate_method": skeleton_anchor.locate_method, "offset": skeleton_anchor.offset, "bulk_count": skeleton_anchor.bulk_count, "pruned_out_of_scope": skeleton_anchor.pruned_count, @@ -350,7 +349,6 @@ def _resolve_pending_tocs( ranges = resolve_hierarchy_page_ranges( resolve_nodes, page_count=toc_scope_end, - page_texts=page_texts, body_pages=toc_body_pages, match_overrides=match_overrides, ) diff --git a/apps/worker/pyproject.toml b/apps/worker/pyproject.toml index d6344112e..021eb6f98 100644 --- a/apps/worker/pyproject.toml +++ b/apps/worker/pyproject.toml @@ -13,10 +13,8 @@ dependencies = [ "pypdf==6.10.2", "beautifulsoup4==4.13.4", "lxml==6.1.0", - "markitdown==0.1.2", "markdownify==1.2.2", "openpyxl==3.1.2", - "pptx2md==2.0.6", "openai==1.93.3", "oss2>=2.18.0", "pillow==12.2.0", diff --git a/apps/worker/requirements.txt b/apps/worker/requirements.txt index 2e22a69a0..8d72aa832 100644 --- a/apps/worker/requirements.txt +++ b/apps/worker/requirements.txt @@ -1,6 +1,6 @@ # This file was autogenerated by uv via the following command: # uv export --no-hashes --no-dev -o requirements.txt --e ../../packages/shared-python +-e ./packages/shared-python # via knowhere-worker-app aiohappyeyeballs==2.6.1 # via aiohttp @@ -49,7 +49,6 @@ beautifulsoup4==4.13.4 # via # knowhere-worker-app # markdownify - # markitdown billiard==4.2.1 # via # celery @@ -80,16 +79,13 @@ cffi==2.0.0 # cryptography # gevent charset-normalizer==3.4.7 - # via - # markitdown - # requests + # via requests click==8.3.3 # via # celery # click-didyoumean # click-plugins # click-repl - # magika click-didyoumean==0.3.1 # via celery click-plugins==1.1.1.2 @@ -110,8 +106,6 @@ cryptography==46.0.7 # authlib # knowhere-worker-app # pyjwt -defusedxml==0.7.1 - # via markitdown distro==1.9.0 # via # openai @@ -198,8 +192,6 @@ lxml==6.1.0 # knowhere-worker-app # python-docx # python-pptx -magika==0.6.2 - # via markitdown makefun==1.16.0 # via fastapi-users mako==1.3.11 @@ -207,10 +199,6 @@ mako==1.3.11 markdown-it-py==4.0.0 # via rich markdownify==1.2.2 - # via - # knowhere-worker-app - # markitdown -markitdown==0.1.2 # via knowhere-worker-app markupsafe==3.0.3 # via mako @@ -225,23 +213,25 @@ networkx==3.6.1 numpy==2.2.6 # via # knowhere-worker-app - # magika # onnxruntime + # opencv-python # pandas # pgvector - # pptx2md # pymupdf-layout # rank-bm25 - # scipy + # rapidocr-onnxruntime + # shapely # tabula-py onnxruntime==1.25.0 # via - # magika # pymupdf-layout + # rapidocr-onnxruntime openai==1.93.3 # via # knowhere-shared # knowhere-worker-app +opencv-python==5.0.0.93 + # via rapidocr-onnxruntime openpyxl==3.1.2 # via knowhere-worker-app opentelemetry-api==1.40.0 @@ -318,14 +308,12 @@ pillow==12.2.0 # via # knowhere-shared # knowhere-worker-app - # pptx2md # python-pptx + # rapidocr-onnxruntime pluggy==1.6.0 # via pytest posthog==7.18.1 # via knowhere-shared -pptx2md==2.0.6 - # via knowhere-worker-app prompt-toolkit==3.0.52 # via click-repl propcache==0.4.1 @@ -348,6 +336,8 @@ psycopg2-binary==2.9.12 # via knowhere-shared pwdlib==0.3.0 # via fastapi-users +pyclipper==1.4.0 + # via rapidocr-onnxruntime pycparser==3.0 ; implementation_name != 'PyPy' # via cffi pycryptodome==3.23.0 @@ -357,7 +347,6 @@ pydantic==2.13.4 # fastapi # knowhere-shared # openai - # pptx2md # pydantic-settings pydantic-core==2.46.4 # via pydantic @@ -405,14 +394,11 @@ python-docx==1.2.0 python-dotenv==1.2.2 # via # knowhere-shared - # magika # pydantic-settings python-multipart==0.0.27 # via fastapi-users python-pptx==1.0.2 - # via - # knowhere-worker-app - # pptx2md + # via knowhere-worker-app pytz==2025.2 # via # knowhere-shared @@ -421,12 +407,13 @@ pyyaml==6.0.2 # via # knowhere-shared # pymupdf-layout + # rapidocr-onnxruntime qstash==3.2.0 # via knowhere-shared rank-bm25==0.2.2 # via knowhere-shared -rapidfuzz==3.14.5 - # via pptx2md +rapidocr-onnxruntime==1.4.4 + # via knowhere-worker-app redis==5.3.1 # via # celery-redbeat @@ -436,7 +423,6 @@ regex==2026.4.4 requests==2.33.0 # via # knowhere-shared - # markitdown # opentelemetry-exporter-otlp-proto-http # oss2 # posthog @@ -444,13 +430,14 @@ rich==15.0.0 # via logfire s3transfer==0.13.1 # via boto3 -scipy==1.17.1 - # via pptx2md +shapely==2.1.2 + # via rapidocr-onnxruntime six==1.17.0 # via # markdownify # oss2 # python-dateutil + # rapidocr-onnxruntime sniffio==1.3.1 # via openai soupsieve==2.8.3 @@ -477,7 +464,7 @@ tqdm==4.67.1 # via # knowhere-worker-app # openai - # pptx2md + # rapidocr-onnxruntime typing-extensions==4.14.1 # via # aiosignal diff --git a/apps/worker/scripts/debug_parse.py b/apps/worker/scripts/debug_parse.py index 6ad95f2de..005d2c04b 100644 --- a/apps/worker/scripts/debug_parse.py +++ b/apps/worker/scripts/debug_parse.py @@ -64,13 +64,21 @@ ) PRODUCTION_OUTPUT_ROOT = Path("~/.knowhere/chengke_kb").expanduser() -AGENT_TRANSIENT_DIRS = ( +PROFILE_TRANSIENT_DIRS = ( "_doc_agent", - "planner_pages", + "coarse_profile_pages", + "calibration_scan", + "calibration_verify", "toc_pages", + "ocr_pages", "inspect_pages", - "verify_pages", + "profile_visuals", + # Legacy dirs from older runs. "agent_visuals", + "planner_pages", + "page_locate_pages", + "verify_pages", + "calibration_inspect", ) # ══════════════════════════════════════════════════════════════════════════════ @@ -302,14 +310,14 @@ def _finalize_output( def _cleanup_agent_transient_dirs(add_dir: str) -> None: """Remove VLM render caches before packaging debug output.""" removed: list[str] = [] - for dirname in AGENT_TRANSIENT_DIRS: + for dirname in PROFILE_TRANSIENT_DIRS: path = os.path.join(add_dir, dirname) if os.path.isdir(path): shutil.rmtree(path) removed.append(dirname) nested_doc_agent = os.path.join(add_dir, "_doc_agent") if os.path.isdir(nested_doc_agent): - for dirname in AGENT_TRANSIENT_DIRS: + for dirname in PROFILE_TRANSIENT_DIRS: path = os.path.join(nested_doc_agent, dirname) if os.path.isdir(path): shutil.rmtree(path) diff --git a/apps/worker/scripts/debug_text_track.py b/apps/worker/scripts/debug_text_track.py index eaae97bae..d80c15bd9 100644 --- a/apps/worker/scripts/debug_text_track.py +++ b/apps/worker/scripts/debug_text_track.py @@ -3,7 +3,7 @@ """Staged text-track document parsing debug script. Supports PDF (shard-aware), DOCX, and MD formats with four breakpoints: - 1. profile — production-aligned DOC_AGENT profile: + 1. profile — production-aligned PROFILE: run_coarse → lightweight (≤MAX) / structural (>MAX) 2. mineru — shard splitting + MinerU extraction 3. hierarchy — heading prediction → merged hierarchy tree @@ -75,7 +75,7 @@ def _stage_profile(pdf_path: str, filename: str, out_dir: Path, model: str | Non from shared.core.config import settings logger.info("=" * 70) - logger.info("🧬 Stage 1: DOC_AGENT profile (production-aligned)") + logger.info("🧬 Stage 1: PROFILE (production-aligned)") logger.info("=" * 70) doc_agent_dir = out_dir / "_doc_agent" @@ -88,11 +88,12 @@ def _stage_profile(pdf_path: str, filename: str, out_dir: Path, model: str | Non output_dir=str(doc_agent_dir), model=vlm_model, settings={ - "planner_model": vlm_model, "vlm_model": vlm_model, "model": settings.HIERARCHY_LLM_MODEL or settings.NORMOL_MODEL, - "toc_profile_enabled": settings.PDF_PROFILE_TOC_ENABLED, - "toc_before_coarse": settings.PDF_PROFILE_TOC_ENABLED, + # Debug PROFILE must exercise the same TOC → run_toc_anchoring path + # as production (PDF_PROFILE_TOC_ENABLED defaults True; keep explicit + # so a local kill-switch .env cannot silently skip TOC). + "toc_profile_enabled": True, }, ) t0 = time.time() @@ -147,6 +148,18 @@ def _stage_profile(pdf_path: str, filename: str, out_dir: Path, model: str | Non shard.shard_index, shard.page_start, shard.page_end, shard.page_end - shard.page_start + 1, shard.anchor_type, ) + pending = list(getattr(anatomy, "pending_skeleton_anchors", None) or []) + if pending: + for record in pending: + toc = record.get("toc") if isinstance(record, dict) else None + logger.info( + " pending toc_range={} relationship={} grafted={}", + (toc or {}).get("toc_range") if isinstance(toc, dict) else None, + record.get("relationship"), + bool(record.get("grafted")), + ) + else: + logger.info(" pending TOC: none") return anatomy, elapsed, { "profile": profile.to_dict() if profile is not None else None, @@ -158,6 +171,19 @@ def _stage_profile(pdf_path: str, filename: str, out_dir: Path, model: str | Non "asset_pages": sum( 1 for feature in coordinator.blackboard.page_features if feature.has_asset ), + "pending_count": len(pending), + "pending": [ + { + "toc_range": (record.get("toc") or {}).get("toc_range") + if isinstance(record.get("toc"), dict) + else None, + "relationship": record.get("relationship"), + "grafted": bool(record.get("grafted")), + } + for record in pending + if isinstance(record, dict) + ], + "skeleton_node_count": len(list(getattr(anatomy, "skeleton_nodes", None) or [])), } @@ -202,7 +228,6 @@ def _load_anatomy_cache(out_dir: Path, pdf_path: str, filename: str): PageLabel( page=int(pl.get("page", 0)), kind=pl.get("kind", "normal"), - confidence=float(pl.get("confidence", 0)), evidence=pl.get("evidence", {}), ) for pl in data.get("page_labels", []) @@ -229,14 +254,29 @@ def _load_anatomy_cache(out_dir: Path, pdf_path: str, filename: str): page_offset=int(s.get("page_offset", 0)), anchor_type=s.get("anchor_type", "forced_max_size"), anchor_evidence=s.get("anchor_evidence", ""), - confidence=float(s.get("confidence", 0) or 0), + toc_hierarchies=( + list(s["toc_hierarchies"]) + if isinstance(s.get("toc_hierarchies"), list) + else None + ), ) for i, s in enumerate(sp.get("shards", [])) ], validation=ValidationReport(valid=True), ), toc_hierarchies=data.get("toc_hierarchies"), - toc_page_offset=data.get("toc_page_offset"), + skeleton_anchor=data.get("skeleton_anchor") + if isinstance(data.get("skeleton_anchor"), dict) + else None, + skeleton_nodes=list(data.get("skeleton_nodes") or []) + if isinstance(data.get("skeleton_nodes"), list) + else None, + pending_skeleton_anchors=list(data.get("pending_skeleton_anchors") or []) + if isinstance(data.get("pending_skeleton_anchors"), list) + else [], + global_signals=dict(data.get("global_signals") or {}) + if isinstance(data.get("global_signals"), dict) + else {}, ) @@ -250,11 +290,10 @@ def _stage_mineru_pdf( ) -> tuple[list[str], float]: """Split PDF into shards and run MinerU extraction only (no heading prediction).""" from app.services.document_parser.formats.pdf.shard_splitter import ( - bin_pack_shards, + map_agent_shards, split_pdf, ) from app.services.document_parser.providers.mineru.pdf_service import parse_via_full - from shared.core.config import settings logger.info("=" * 70) logger.info("🔄 Stage 2: Shard splitting + MinerU extraction") @@ -267,8 +306,7 @@ def _stage_mineru_pdf( if anatomy.toc_result and anatomy.toc_result.toc_pages: toc_pages = set(anatomy.toc_result.toc_pages) - max_pages = int(os.environ.get("MAX_PDF_PAGE_LIMIT", getattr(settings, "MAX_PDF_PAGE_LIMIT", 200))) - merged_shards = bin_pack_shards(agent_shards, max_pages=max_pages) + merged_shards = map_agent_shards(agent_shards) logger.info(" {} agent shards → {} MinerU shards", len(agent_shards), len(merged_shards)) work_dir = str(out_dir / "_shards") @@ -320,9 +358,7 @@ def _stage_hierarchy_pdf( merge_images, merge_shard_lines, ) - from app.services.document_parser.formats.pdf.shard_splitter import bin_pack_shards - from app.services.document_agent.tools.propose_shard_plan import split_toc_for_shard - from shared.core.config import settings + from app.services.document_parser.formats.pdf.shard_splitter import map_agent_shards logger.info("=" * 70) logger.info("🔬 Stage 3: Per-shard heading prediction → merged hierarchy") @@ -330,10 +366,8 @@ def _stage_hierarchy_pdf( t0 = time.time() agent_shards = anatomy.shard_plan.shards - toc_hierarchies = anatomy.toc_hierarchies - max_pages = int(os.environ.get("MAX_PDF_PAGE_LIMIT", getattr(settings, "MAX_PDF_PAGE_LIMIT", 200))) - merged_shards = bin_pack_shards(agent_shards, max_pages=max_pages) + merged_shards = map_agent_shards(agent_shards) work_dir = out_dir / "_shards" @@ -358,14 +392,8 @@ def _predict_shard(shard_idx: int, shard_out_dir: str) -> list[str]: md_lines = merge_html_tables(md_lines) is_first = shard_idx == 0 - shard = merged_shards[shard_idx] - shard_toc = ( - toc_hierarchies if is_first - else split_toc_for_shard( - toc_hierarchies, shard.page_start, shard.page_end, - offset_override=getattr(anatomy, "toc_page_offset", None), - ) - ) + agent_shard = agent_shards[shard_idx] + shard_toc = getattr(agent_shard, "toc_hierarchies", None) lines_with_heading = eval_md_headings( md_lines, @@ -721,6 +749,7 @@ def main() -> int: trace["stages"].setdefault("profile", {})["shard_count"] = len( anatomy.shard_plan.shards ) + trace["token_usage"] = get_current_token_tracker() _write_json(out_dir / "trace.json", trace) logger.info("⏸️ Stopped at profile → {}", out_dir) return 0 @@ -738,6 +767,7 @@ def main() -> int: logger.info("⏩ Reusing cached MinerU shard dirs") if args.stop_at == "mineru": + trace["token_usage"] = get_current_token_tracker() _write_json(out_dir / "trace.json", trace) logger.info("⏸️ Stopped at mineru → {}", out_dir) return 0 @@ -760,6 +790,7 @@ def main() -> int: } if args.stop_at == "hierarchy": + trace["token_usage"] = get_current_token_tracker() _write_json(out_dir / "trace.json", trace) logger.info("⏸️ Stopped at hierarchy → {}", out_dir) return 0 diff --git a/apps/worker/scripts/page_memory/_debug_pm_shared.py b/apps/worker/scripts/page_memory/_debug_pm_shared.py index 173a18ffa..88aa9e62b 100644 --- a/apps/worker/scripts/page_memory/_debug_pm_shared.py +++ b/apps/worker/scripts/page_memory/_debug_pm_shared.py @@ -230,28 +230,25 @@ def build_ctx( asset_extraction_enabled: bool = False, ): from app.services.document_agent.manifest import ToolContext - from app.services.document_agent.state import AgentBlackboard - from app.services.document_agent.budget import BudgetTracker + from app.services.document_agent.state import ProfileBlackboard - blackboard = AgentBlackboard() + blackboard = ProfileBlackboard() blackboard.page_count = page_count blackboard.page_full_text_cache = dict(page_texts) vmodel = vlm_model or os.environ.get("IMAGE_MODEL") reason_model = os.environ.get("PAGE_LOCATE_REASON_MODEL") or os.environ.get("NORMOL_MODEL") - budget = BudgetTracker(plan_budget=50000, visual_budget=200000) return ToolContext( pdf_path=pdf_path, job_id=job_id, blackboard=blackboard, - budget=budget, trace=None, output_dir=str(out_dir / "_doc_agent"), settings={ "vlm_model": vmodel, "model": reason_model, - "agent_png_dpi": os.environ.get("AGENT_PNG_DPI", "144"), + "profile_png_dpi": os.environ.get("AGENT_PNG_DPI", "144"), }, ) @@ -276,8 +273,6 @@ def resolve_anatomy_cache_path(out_dir: Path) -> Path: def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): from app.services.document_agent.manifest import ( - H1BoundaryResult, - H1Candidate, PageAnatomyMap, PageFeature, PageLabel, @@ -292,7 +287,6 @@ def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): logger.info(f"⏩ Reusing cached anatomy: {cache_path}") data = json.loads(cache_path.read_text(encoding="utf-8")) toc = data.get("toc_result") or {} - h1 = data.get("h1_result") or {} sp = data.get("shard_plan") or {} page_features = [] @@ -316,7 +310,6 @@ def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): page_labels.append(PageLabel( page=int(pl.get("page", 0)), kind=pl.get("kind", "normal"), - confidence=float(pl.get("confidence", 0)), evidence=pl.get("evidence", {}), )) @@ -328,7 +321,7 @@ def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): TocAnchorPage( page=int(candidate.get("page", 0)), png_path=str(candidate.get("png_path") or ""), - source=candidate.get("source", "text_scan"), + source="text_scan", ) ) evidence = [] @@ -339,11 +332,14 @@ def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): TocEvidence( page_index=int(item.get("page_index", 0)), source=str(item.get("source") or ""), - confidence=float(item.get("confidence", 0) or 0), reason=str(item.get("reason") or ""), ) ) + method = toc.get("method", "none") + if method not in {"vlm_batch", "pdf_outline", "none"}: + method = "none" + return PageAnatomyMap( job_id=data.get("job_id", job_id), file_path=data.get("file_path", pdf_path), @@ -354,22 +350,10 @@ def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): toc_pages=list(toc.get("toc_pages", [])), candidates=candidates, evidence=evidence, - method=toc.get("method", "none"), + method=method, # type: ignore[arg-type] notes=str(toc.get("notes") or ""), failure_kind=toc.get("failure_kind", "none"), ), - h1_result=H1BoundaryResult( - h1_candidates=[ - H1Candidate( - title=c.get("title", ""), - page=int(c.get("page", 0)), - confidence=float(c.get("confidence", 0) or 0), - matched_line=c.get("matched_line", ""), - source=c.get("source", "none"), - ) - for c in h1.get("h1_candidates", []) - ], - ), shard_plan=ShardPlan( enabled=bool(sp.get("enabled", False)), reason=sp.get("reason", "not_needed"), @@ -381,13 +365,30 @@ def load_anatomy_cache(cache_path: Path, pdf_path: str, job_id: str): page_offset=int(s.get("page_offset", 0)), anchor_type=s.get("anchor_type", "forced_max_size"), anchor_evidence=s.get("anchor_evidence", ""), - confidence=float(s.get("confidence", 0) or 0), + toc_hierarchies=( + list(s["toc_hierarchies"]) + if isinstance(s.get("toc_hierarchies"), list) + else None + ), ) for i, s in enumerate(sp.get("shards", [])) ], validation=ValidationReport(valid=True), ), toc_hierarchies=data.get("toc_hierarchies"), + document_profile=_document_profile_from_dict(data.get("document_profile")), + skeleton_anchor=data.get("skeleton_anchor") + if isinstance(data.get("skeleton_anchor"), dict) + else None, + skeleton_nodes=list(data.get("skeleton_nodes") or []) + if isinstance(data.get("skeleton_nodes"), list) + else None, + pending_skeleton_anchors=list(data.get("pending_skeleton_anchors") or []) + if isinstance(data.get("pending_skeleton_anchors"), list) + else [], + global_signals=dict(data.get("global_signals") or {}) + if isinstance(data.get("global_signals"), dict) + else {}, ) @@ -405,10 +406,10 @@ def run_profile( """Run page-memory profile exactly like production ``memory_service.run``. Uses ``profile_document(..., skip_shard_plan=True, oversized_policy="page_memory")`` - so coarse → anatomy matches the live track (no ReAct shard planning). + so coarse → anatomy matches the live track (no LLM shard planning). - ``skip_toc_anchoring=True`` stops after TOC extract + link attach (legacy - monolithic helper). Prefer staged debug: Stage-0 bootstrap then Stage-1 TOC. + ``skip_toc_anchoring=True`` stops after TOC extract (legacy monolithic + helper). Prefer staged debug: Stage-0 bootstrap then Stage-1 TOC. """ from app.services.document_parser.profiling.doc_profiler import profile_document from shared.core.config import settings @@ -417,7 +418,7 @@ def run_profile( logger.info(f"🧬 DOC_PROFILE (page_memory, monolithic) — {job_id}") logger.info("=" * 70) if skip_toc_anchoring: - logger.info(" skip_toc_anchoring=True (TOC + links only; no calibration)") + logger.info(" skip_toc_anchoring=True (TOC extract only; no calibration)") previous_image_model = settings.IMAGE_MODEL if model: @@ -466,8 +467,6 @@ def run_profile( logger.info(f" page_count={anatomy.page_count}") logger.info(f" toc_pages={anatomy.toc_result.toc_pages}") logger.info(f" has_asset_pages={asset_pages}/{anatomy.page_count}") - if anatomy.h1_result: - logger.info(f" h1_candidates={len(anatomy.h1_result.h1_candidates)}") logger.info( " shard_plan.enabled={} shards={}", anatomy.shard_plan.enabled, @@ -494,7 +493,6 @@ def _build_debug_coordinator( agent_output_dir = out_dir / "_doc_agent" agent_output_dir.mkdir(parents=True, exist_ok=True) merged = { - "planner_model": settings.IMAGE_MODEL, "vlm_model": settings.IMAGE_MODEL, "toc_profile_enabled": True, "model": settings.HIERARCHY_LLM_MODEL or settings.NORMOL_MODEL, @@ -564,7 +562,6 @@ def _page_labels_from_dicts(rows: list[Any]) -> list[Any]: PageLabel( page=int(pl.get("page", 0)), kind=pl.get("kind", "normal"), - confidence=float(pl.get("confidence", 0)), evidence=dict(pl.get("evidence") or {}), ) ) @@ -632,7 +629,6 @@ def load_stage0_into_coordinator(coordinator, out_dir: Path) -> None: # Stage-1 owns TOC + assets from here. bb.toc_result = None bb.toc_hierarchies = None - bb.toc_page_offset = None bb.skeleton_anchor = None bb.skeleton_nodes = None bb.pending_skeleton_anchors = [] @@ -720,7 +716,7 @@ def run_stage1_toc( out_dir: Path, model: str | None, ): - """Production-aligned Stage-1: Find → extract → link attach (no calibration). + """Production-aligned Stage-1: Find → extract (no calibration). Resumes Stage-0 blackboard (including asset probe). Skips ``run_toc_anchoring`` (Stage-2). Does not re-run asset probe. @@ -734,7 +730,7 @@ def run_stage1_toc( from shared.core.config import settings logger.info("=" * 70) - logger.info(f"🧬 Stage 1: TOC FIND → EXTRACT → LINK — {job_id}") + logger.info(f"🧬 Stage 1: TOC FIND → EXTRACT — {job_id}") logger.info("=" * 70) previous_image_model = settings.IMAGE_MODEL @@ -756,6 +752,11 @@ def run_stage1_toc( persist_anatomy_map(coordinator.ctx, {}) profile_path = out_dir / DOC_PROFILE_FILENAME write_debug_json(profile_path, anatomy.to_dict()) + # Canonical profile is at package root; drop nested duplicate. + try: + (out_dir / "_doc_agent" / "anatomy_map.json").unlink() + except FileNotFoundError: + pass update_pipeline_state( pipeline_state_path(out_dir), stage=1, @@ -1038,10 +1039,22 @@ def remove_legacy_doc_agent_artifacts( doc_agent_dir: Path, *, include_stage2: bool = False, + keep_resume_cache: bool = True, ) -> None: + """Drop nested doc-agent clutter; keep resume + pipeline history by default. + + Canonical package artifacts live at ``page_memory/`` root + (``doc_profile.json``, ``trace.json``). Nested ``anatomy_map.json`` and + calibration page PNGs are duplicates / inspect leftovers. + """ + import shutil + names = { "parser_profile.json", "toc_hierarchies.json", + "anatomy_map.json", + "trace.json", + "doc_profile.json", } if include_stage2: names.update( @@ -1052,13 +1065,20 @@ def remove_legacy_doc_agent_artifacts( "stage2_state.json", } ) + if not keep_resume_cache: + names.update( + { + STAGE0_STATE_NAME, + PAGE_TEXT_CACHE_NAME, + "stage_costs.json", + } + ) for name in names: (doc_agent_dir / name).unlink(missing_ok=True) - legacy_preview_dir = doc_agent_dir / "coarse_assets" - if legacy_preview_dir.is_dir(): - import shutil - - shutil.rmtree(legacy_preview_dir) + for dirname in ("coarse_assets", "calibration_inspect"): + legacy_dir = doc_agent_dir / dirname + if legacy_dir.is_dir(): + shutil.rmtree(legacy_dir) (doc_agent_dir / "coarse_assets.html").unlink(missing_ok=True) @@ -1394,6 +1414,7 @@ def stop_with_trace( summary=summary, ) remove_nested_doc_agent_trace(out_dir) + remove_legacy_doc_agent_artifacts(out_dir / "_doc_agent", include_stage2=True) maybe_purge_debug_visuals(out_dir) return 0 diff --git a/apps/worker/scripts/page_memory/debug_pm_stage0_bootstrap.py b/apps/worker/scripts/page_memory/debug_pm_stage0_bootstrap.py index 6dc260a31..45ea96318 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage0_bootstrap.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage0_bootstrap.py @@ -6,7 +6,7 @@ bootstrap → coarse VLM → text scan → asset probe → persist stage0_state + page_full_text_cache -TOC Find / extract / links belong to Stage 1 +TOC Find / extract belong to Stage 1 (``debug_pm_stage1_hierarchy.py``). Calibration belongs to Stage 2. Usage: diff --git a/apps/worker/scripts/page_memory/debug_pm_stage1_hierarchy.py b/apps/worker/scripts/page_memory/debug_pm_stage1_hierarchy.py index 664a93616..14e9e3f37 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage1_hierarchy.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage1_hierarchy.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # ruff: noqa: E402 -"""Stage 1: TOC Find → extract → link attach (no calibration). +"""Stage 1: TOC Find → extract (no calibration). Resumes Stage-0 blackboard (``stage0_state.json`` + ``page_full_text_cache.json``, including asset-probe ``has_asset`` flags) and runs the production TOC segment: - find.toc_anchor_pages → extract.toc_with_boundaries → attach links + find.toc_anchor_pages → extract.toc_with_boundaries → persist doc_profile.json Requires Stage 0 first: @@ -53,9 +53,8 @@ def _count_hierarchy_keys(tree: dict) -> int: return total -def _count_linked_entries(toc_hierarchies: list | None) -> tuple[int, int]: +def _count_toc_entries(toc_hierarchies: list | None) -> int: total = 0 - linked = 0 for region in toc_hierarchies or []: if not isinstance(region, dict): continue @@ -63,28 +62,24 @@ def _count_linked_entries(toc_hierarchies: list | None) -> tuple[int, int]: if not isinstance(entries, list): continue for entry in entries: - if not isinstance(entry, dict): - continue - total += 1 - link = entry.get("link") - if isinstance(link, dict) and link.get("physical_page") is not None: - linked += 1 - return linked, total + if isinstance(entry, dict): + total += 1 + return total def main() -> int: - parser = base_argparser("Stage 1: TOC Find → extract → link (no calibration)") + parser = base_argparser("Stage 1: TOC Find → extract (no calibration)") parser.add_argument( "--reuse-anatomy", action="store_true", - help="Reuse cached Stage-1 doc_profile.json (skip Find/extract/link)", + help="Reuse cached Stage-1 doc_profile.json (skip Find/extract)", ) args = parser.parse_args() pdf_path, filename, out_dir = resolve_paths(args) logger.info("█" * 70) - logger.info(f" STAGE 1: TOC FIND → EXTRACT → LINK — {filename}") + logger.info(f" STAGE 1: TOC FIND → EXTRACT — {filename}") logger.info(f" OUTPUT: {out_dir}") logger.info("█" * 70) @@ -110,7 +105,7 @@ def main() -> int: profile_source = "stage0_resume_toc" page_count = anatomy.page_count - linked, total = _count_linked_entries(list(anatomy.toc_hierarchies or [])) + total = _count_toc_entries(list(anatomy.toc_hierarchies or [])) record_stage( trace_stages, "toc", @@ -119,7 +114,6 @@ def main() -> int: "source": profile_source, "toc_pages": anatomy.toc_result.toc_pages, "toc_entries": total, - "toc_entries_with_link": linked, "skip_toc_anchoring": True, "skeleton_anchor": getattr(anatomy, "skeleton_anchor", None), }, @@ -130,7 +124,7 @@ def main() -> int: logger.info("=" * 70) logger.info("🧠 TOC hierarchy (Stage-1 debug dump)") logger.info("=" * 70) - logger.info(" TOC entries with link: {}/{}", linked, total) + logger.info(" TOC entries: {}", total) hierarchy_tree = toc_hierarchies_to_hierarchy_tree(anatomy.toc_hierarchies) toc_path = write_toc_hierarchy_artifact( @@ -140,7 +134,6 @@ def main() -> int: "source": "toc_hierarchies_raw", "region_count": len(list(anatomy.toc_hierarchies or [])), "hierarchy_key_count": _count_hierarchy_keys(hierarchy_tree), - "toc_entries_with_link": linked, "toc_entries": total, }, ) diff --git a/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py b/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py index a659e28c8..32140c90f 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py @@ -1,10 +1,14 @@ #!/usr/bin/env python3 # ruff: noqa: E402 -"""Stage 2: Calibration over Stage-1 TOC ``doc_profile.json``. +"""Stage 2: Production ``run_toc_anchoring`` over Stage-1 TOC. -Reads Stage-1 output (TOC hierarchies + optional ``link.physical_page``), -runs calibration (Agent Phase-1 + Phase-2 completion), and writes -``skeleton_anchor`` / ``toc_page_offset`` back onto ``doc_profile.json``. +Same PROFILE anchoring path as production PAGE/TEXT: + + select primary/pending → calibrate (Agent Phase-1 + Phase-2) → + classify contained/parallel → graft contained → write skeleton_* + +Also resolves coarse skeletons (C4 resolve-only) into pipeline state so +Stage 3 can resume without re-anchoring. No fine hierarchy. Requires Stage 0 → Stage 1 first: uv run python scripts/page_memory/debug_pm_stage0_bootstrap.py --file ... @@ -13,16 +17,14 @@ Usage: cd apps/worker uv run python scripts/page_memory/debug_pm_stage2_calibration.py --file /path/to/doc.pdf - uv run python scripts/page_memory/debug_pm_stage2_calibration.py --file /path/to/doc.pdf --no-links """ from __future__ import annotations -import json -import os import sys import time from pathlib import Path as _Path +from typing import Any sys.path.insert(0, str(_Path(__file__).resolve().parent)) @@ -30,47 +32,69 @@ from _debug_pm_shared import ( TokenCostTracker, + _build_debug_coordinator, + _serialize_skeletons, base_argparser, load_anatomy_cache, - resolve_anatomy_cache_path, + load_stage0_into_coordinator, + page_text_cache_path, pipeline_state_path, record_stage, require_file, + resolve_anatomy_cache_path, resolve_paths, + stage0_state_path, stop_with_trace, update_pipeline_state, write_debug_json, ) +def _pending_summary(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for record in records: + toc = record.get("toc") if isinstance(record, dict) else None + rows.append( + { + "toc_range": (toc or {}).get("toc_range") if isinstance(toc, dict) else None, + "relationship": record.get("relationship"), + "grafted": bool(record.get("grafted")), + "graft_events": len(list(record.get("graft") or [])), + "has_nodes": bool(record.get("nodes")), + } + ) + return rows + + def main() -> int: - parser = base_argparser("Stage 2: Calibration SubAgent") - parser.add_argument( - "--no-links", - action="store_true", - help="Strip link.physical_page from TOC entries before the agent runs", - ) - parser.add_argument( - "--max-rounds", - type=int, - default=16, - help="Max ReAct rounds per TOC region", + parser = base_argparser( + "Stage 2: Production run_toc_anchoring (calibrate + classify + graft)" ) args = parser.parse_args() - from app.services.document_agent.agents.calibration import ( - run_calibration_for_all_regions, - ) - from app.services.document_agent.pdf_text import read_page_texts from app.services.document_agent.persist import DOC_PROFILE_FILENAME + from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring + from app.services.document_agent.validators import single_shard_plan + from app.services.page_memory.skeleton_extractor import extract_section_skeletons + from shared.core.config import settings pdf_path, filename, out_dir = resolve_paths(args) anatomy_cache = resolve_anatomy_cache_path(out_dir) + require_file( + stage0_state_path(out_dir), + hint=( + "Run Stage 0 first:\n" + " uv run python scripts/page_memory/debug_pm_stage0_bootstrap.py --file ..." + ), + ) + require_file( + page_text_cache_path(out_dir), + hint="Stage-0 page_full_text_cache.json missing; re-run Stage 0", + ) require_file( anatomy_cache, hint=( - "Run Stage 0 then Stage 1 first:\n" - " uv run python scripts/page_memory/debug_pm_stage0_bootstrap.py --file ...\n" + "Run Stage 1 first:\n" " uv run python scripts/page_memory/debug_pm_stage1_hierarchy.py --file ..." ), ) @@ -80,66 +104,65 @@ def main() -> int: hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or []) logger.info("█" * 70) - logger.info(f" STAGE 2: CALIBRATION SUBAGENT — {filename}") + logger.info(f" STAGE 2: run_toc_anchoring (production) — {filename}") logger.info(f" OUTPUT: {out_dir}") - logger.info(f" no_links={bool(args.no_links)} regions={len(hierarchies)}") + logger.info(" regions={}", len(hierarchies)) logger.info("█" * 70) t_start = time.time() trace_stages: list[dict] = [] token_cost_tracker = TokenCostTracker() - # Production null-page locate needs page texts (same as C4). - page_texts = read_page_texts(pdf_path, list(range(1, page_count + 1)), timeout=300) - body_pages = sorted(page_texts.keys()) - logger.info( - " read {} pages, {} non-empty", - len(page_texts), - sum(1 for text in page_texts.values() if str(text).strip()), - ) - - vlm_model = args.vlm_model or os.environ.get("IMAGE_MODEL") or "" - planner_model = ( - args.model - or os.environ.get("HIERARCHY_LLM_MODEL") - or os.environ.get("NORMOL_MODEL") - or vlm_model + previous_image_model = settings.IMAGE_MODEL + try: + coordinator = _build_debug_coordinator( + pdf_path=pdf_path, + job_id=filename, + out_dir=out_dir, + model=args.model, + settings_extra={ + # Stage-1 already extracted TOC; Stage-2 only anchors. + "skip_toc_anchoring": False, + }, + ) + load_stage0_into_coordinator(coordinator, out_dir) + bb = coordinator.blackboard + bb.toc_result = anatomy.toc_result + bb.toc_hierarchies = hierarchies + bb.shard_plan = anatomy.shard_plan or single_shard_plan(page_count) + bb.skeleton_anchor = None + bb.skeleton_nodes = None + bb.pending_skeleton_anchors = [] + + run_toc_anchoring(coordinator.ctx) + + # Persist the same PROFILE fields production anatomy carries. + from app.services.document_agent.persist import build_anatomy_map + + bb.shard_plan = bb.shard_plan or single_shard_plan(page_count) + anchored = build_anatomy_map(coordinator.ctx) + profile_path = out_dir / DOC_PROFILE_FILENAME + write_debug_json(profile_path, anchored.to_dict()) + try: + (out_dir / "_doc_agent" / "anatomy_map.json").unlink() + except FileNotFoundError: + pass + + page_texts = dict(bb.page_full_text_cache or {}) + skeletons = extract_section_skeletons( + anatomy=anchored, + filename=filename, + page_texts=page_texts, + ) + finally: + if args.model: + settings.IMAGE_MODEL = previous_image_model + + pending_records = list( + getattr(coordinator.blackboard, "pending_skeleton_anchors", None) or [] ) - - doc_agent_dir = out_dir / "_doc_agent" - doc_agent_dir.mkdir(parents=True, exist_ok=True) - - calibration = run_calibration_for_all_regions( - pdf_path=pdf_path, - page_count=page_count, - toc_hierarchies=hierarchies, - output_dir=str(doc_agent_dir), - vlm_model=vlm_model, - planner_model=planner_model, - no_links=bool(args.no_links), - max_rounds=max(1, int(args.max_rounds)), - page_texts=page_texts, - body_pages=body_pages, - ) - - # Production-compatible core fields. - skeleton_anchor = { - "offset": calibration.get("offset"), - "offset_status": calibration.get("offset_status"), - "match_overrides": calibration.get("match_overrides") or {}, - "null_page_report": calibration.get("null_page_report") or [], - "bulk_count": calibration.get("bulk_count") or 0, - "pruned_count": calibration.get("pruned_count") or 0, - "locate_agent": calibration.get("locate_agent") or "offset_only", - } - - profile_path = out_dir / DOC_PROFILE_FILENAME - payload = json.loads(profile_path.read_text(encoding="utf-8")) - payload["skeleton_anchor"] = skeleton_anchor - payload["calibration"] = calibration - if calibration.get("offset") is not None: - payload["toc_page_offset"] = calibration.get("offset") - write_debug_json(profile_path, payload) + skeleton_anchor = getattr(coordinator.blackboard, "skeleton_anchor", None) or {} + pending_summary = _pending_summary(pending_records) state_path = pipeline_state_path(out_dir) update_pipeline_state( @@ -148,36 +171,49 @@ def main() -> int: document={ "source_file_name": filename, "page_count": page_count, - "anatomy_path": str(anatomy_cache), + "anatomy_path": str(profile_path), + }, + payload={ + "skeleton_anchor": skeleton_anchor, + "skeleton_nodes": list( + getattr(coordinator.blackboard, "skeleton_nodes", None) or [] + ), + "pending_skeleton_anchors": pending_records, + "pending_summary": pending_summary, + "skeletons": _serialize_skeletons(skeletons), }, - payload={"skeleton_anchor": skeleton_anchor, "calibration": calibration}, ) record_stage( trace_stages, - "calibration", + "toc_anchoring", page_info={"page_count": page_count}, variables={ - "status": calibration.get("status"), - "failure_kind": calibration.get("failure_kind"), - "offset": calibration.get("offset"), - "offset_status": calibration.get("offset_status"), - "bulk_count": calibration.get("bulk_count"), - "locate_agent": calibration.get("locate_agent"), - "regime_count": len(calibration.get("regimes") or []), - "tool_calls": calibration.get("tool_calls"), - "no_links": bool(args.no_links), + "offset": skeleton_anchor.get("offset") + if isinstance(skeleton_anchor, dict) + else None, + "offset_status": skeleton_anchor.get("offset_status") + if isinstance(skeleton_anchor, dict) + else None, + "bulk_count": skeleton_anchor.get("bulk_count") + if isinstance(skeleton_anchor, dict) + else None, + "locate_method": skeleton_anchor.get("locate_method") + if isinstance(skeleton_anchor, dict) + else None, + "skeleton_count": len(skeletons), + "pending": pending_summary, }, ) - token_cost_tracker.snapshot_stage("calibration") + token_cost_tracker.snapshot_stage("toc_anchoring") elapsed = time.time() - t_start logger.info( - "✅ Stage 2 done status={} offset={} bulk={} locate={} in {:.1f}s → {}", - calibration.get("status"), - calibration.get("offset"), - calibration.get("bulk_count"), - calibration.get("locate_agent"), + "✅ Stage 2 done offset={} bulk={} skeletons={} pending={} in {:.1f}s → {}", + skeleton_anchor.get("offset") if isinstance(skeleton_anchor, dict) else None, + skeleton_anchor.get("bulk_count") if isinstance(skeleton_anchor, dict) else None, + len(skeletons), + pending_summary, elapsed, profile_path, ) @@ -190,6 +226,7 @@ def main() -> int: pipeline_stage=2, elapsed_s=elapsed, token_cost_tracker=token_cost_tracker, + extra_summary={"pending": pending_summary, "skeleton_count": len(skeletons)}, ) diff --git a/apps/worker/scripts/page_memory/debug_pm_stage3_coarse_scope.py b/apps/worker/scripts/page_memory/debug_pm_stage3_coarse_scope.py index e0a953ebe..9ed64bde3 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage3_coarse_scope.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage3_coarse_scope.py @@ -6,7 +6,8 @@ directories with ``skeletons.json`` (meta + coarse nodes) plus empty ``page_tags.json`` / ``assets.json`` placeholders for later stages. -Requires Stage 2 output: _doc_agent/pipeline_state.json, doc_profile.json +Requires Stage 2 output: _doc_agent/pipeline_state.json (with skeletons), +doc_profile.json (after production ``run_toc_anchoring``). Usage: cd apps/worker @@ -129,7 +130,7 @@ def main() -> int: end_page=page_count, title="Root", parent_path=filename, - evidence={"source": "fallback_root", "confidence": 0.0}, + evidence={"source": "fallback_root"}, ) coarse_scopes = [ { diff --git a/apps/worker/scripts/page_memory/debug_pm_stage5_assets.py b/apps/worker/scripts/page_memory/debug_pm_stage5_assets.py index 7241fda04..5344ea08e 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage5_assets.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage5_assets.py @@ -236,7 +236,6 @@ def main() -> int: rendered_pages=asset_rendered, output_dir=str(out_dir), model_name=asset_model, - budget=None, max_pages=asset_max_pages, confidence_threshold=get_asset_confidence_threshold(), summary_enabled=summary_enabled, diff --git a/apps/worker/tests/contract/test_body_boundary_contract.py b/apps/worker/tests/contract/test_body_boundary_contract.py new file mode 100644 index 000000000..3512c9f89 --- /dev/null +++ b/apps/worker/tests/contract/test_body_boundary_contract.py @@ -0,0 +1,87 @@ +"""Contracts for TOC-derived body-boundary helpers used by TEXT-TRACK.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_parser.structure.body_boundary import ( + extract_level1_titles, + find_first_body_boundary, +) +from app.services.document_parser.structure.layout_parser import ( + _supports_multi_toc_zones, +) + + +def test_extract_level1_titles_reads_toc_with_level_not_toc_tree() -> None: + titles = extract_level1_titles( + [ + { + "toc_range": [1, 20], + "toc_range_unit": "page", + "source": "calibrated_shard_split", + # Legacy-shaped tree must be ignored: production slices omit it. + "toc_tree": {"Ignore Me": {}}, + "toc_with_level": [ + {"heading": "1. Overview", "level": 1}, + {"heading": "1.1 Scope", "level": 2}, + {"heading": "2. Requirements", "level": 1}, + ], + } + ] + ) + assert titles == ["Overview", "Requirements"] + + +def test_extract_level1_titles_ignores_empty_or_non_list_payloads() -> None: + assert extract_level1_titles([]) == [] + assert extract_level1_titles([{"toc_with_level": None}]) == [] + assert extract_level1_titles([{"toc_with_level": "| heading | level |"}]) == [] + assert extract_level1_titles([{"toc_tree": {"Only Tree": {}}}]) == [] + + +def test_find_first_body_boundary_matches_cleaned_level1_in_md_lines() -> None: + boundary = find_first_body_boundary( + [ + "Cover Page", + "Legal Notice", + "# 1. Overview", + "Body text", + ], + ["Overview"], + ) + assert boundary == 2 + + +def test_multi_toc_zones_are_isolated_from_profile_page_coordinates() -> None: + page_tocs = [ + {"toc_range": [1, 10], "toc_range_unit": "page"}, + {"toc_range": [11, 20], "toc_range_unit": "page"}, + ] + line_tocs = [ + {"toc_range": [1, 10]}, + {"toc_range": [30, 40]}, + ] + + assert not _supports_multi_toc_zones( + page_tocs, + doc_type="md", + smart_parse=True, + ) + assert _supports_multi_toc_zones( + line_tocs, + doc_type="md", + smart_parse=True, + ) + assert _supports_multi_toc_zones( + line_tocs, + doc_type="docx", + smart_parse=True, + ) diff --git a/apps/worker/tests/contract/test_calibration_phase1_contract.py b/apps/worker/tests/contract/test_calibration_phase1_contract.py new file mode 100644 index 000000000..fdbe2c247 --- /dev/null +++ b/apps/worker/tests/contract/test_calibration_phase1_contract.py @@ -0,0 +1,223 @@ +"""Contract tests for deterministic calibration Phase-1 (regimes + scan).""" + +from __future__ import annotations + +import os +from typing import Any + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +import pytest + +from app.services.document_agent.calibration import phase1 as phase1_module +from app.services.document_agent.calibration.phase1 import ( + PROBES_PER_REGIME, + run_calibration_phase1, +) +from app.services.document_agent.calibration.scan import TitleScanResult +from app.services.document_agent.calibration.types import ( + FAILURE_NO_OFFSET, + FAILURE_PAGE_COUNT_MISSING, + FAILURE_TOC_EMPTY, +) +from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.state import ProfileBlackboard + + +def _ctx(page_count: int = 60) -> ToolContext: + blackboard = ProfileBlackboard() + blackboard.page_count = page_count + return ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="job-phase1", + blackboard=blackboard, + trace=None, + settings={"vlm_model": "test-vlm"}, + ) + + +def _hierarchy(entries: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [{"toc_range": [1, 3], "toc_with_level": entries}] + + +class _FakeScan: + """Answers each title from ``hits``; records probe order.""" + + def __init__(self, hits: dict[str, int]) -> None: + self.hits = hits + self.calls: list[tuple[str, int]] = [] + + def __call__( + self, + *, + ctx: ToolContext, + title: str, + start_page: int, + page_count: int, + **kwargs: Any, + ) -> TitleScanResult: + self.calls.append((title, start_page)) + found_page = self.hits.get(title) + return TitleScanResult( + title=title, + found=found_page is not None, + found_page=found_page, + scanned_pages=[start_page], + next_start=start_page + 1, + ) + + +@pytest.fixture +def patch_scan(monkeypatch: pytest.MonkeyPatch): + def _apply(fake: _FakeScan) -> _FakeScan: + monkeypatch.setattr(phase1_module, "scan_title_forward", fake) + return fake + + return _apply + + +def test_offset_is_found_page_minus_printed(patch_scan) -> None: + fake = patch_scan(_FakeScan({"Chapter 1": 15})) + hierarchies = _hierarchy( + [ + {"heading": "Chapter 1", "page_number": "10", "level": 1}, + {"heading": "Chapter 2", "page_number": "20", "level": 1}, + ] + ) + + result = run_calibration_phase1( + ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60 + ) + + assert result.status == "ok" + assert [(r.kind, r.offset) for r in result.regimes] == [("decimal", 5)] + # toc_range=[1, 3] → scan starts at page after TOC end. + assert fake.calls == [("Chapter 1", 4)] + + +def test_first_hit_stops_the_regime(patch_scan) -> None: + fake = patch_scan(_FakeScan({"Chapter 1": 15, "Chapter 2": 25})) + hierarchies = _hierarchy( + [ + {"heading": "Chapter 1", "page_number": "10", "level": 1}, + {"heading": "Chapter 2", "page_number": "20", "level": 1}, + ] + ) + + run_calibration_phase1(ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60) + + assert fake.calls == [("Chapter 1", 4)] + + +def test_second_probe_runs_when_the_first_misses(patch_scan) -> None: + fake = patch_scan(_FakeScan({"Chapter 2": 25})) + hierarchies = _hierarchy( + [ + {"heading": "Chapter 1", "page_number": "10", "level": 1}, + {"heading": "Chapter 2", "page_number": "20", "level": 1}, + {"heading": "Chapter 3", "page_number": "30", "level": 1}, + ] + ) + + result = run_calibration_phase1( + ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60 + ) + + assert fake.calls == [("Chapter 1", 4), ("Chapter 2", 4)] + assert [r.offset for r in result.regimes] == [5] + + +def test_probe_count_per_regime_is_capped(patch_scan) -> None: + fake = patch_scan(_FakeScan({})) + hierarchies = _hierarchy( + [ + {"heading": f"Chapter {i}", "page_number": str(i), "level": 1} + for i in range(1, 8) + ] + ) + + result = run_calibration_phase1( + ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60 + ) + + assert len(fake.calls) == PROBES_PER_REGIME + assert result.status == "failed" + assert result.failure_kind == FAILURE_NO_OFFSET + + +def test_roman_and_decimal_regimes_calibrate_independently(patch_scan) -> None: + fake = patch_scan(_FakeScan({"Preface": 4, "Chapter 1": 15})) + hierarchies = _hierarchy( + [ + {"heading": "Preface", "page_number": "ii", "level": 1}, + {"heading": "Foreword", "page_number": "iv", "level": 1}, + {"heading": "Chapter 1", "page_number": "10", "level": 1}, + {"heading": "Chapter 2", "page_number": "20", "level": 1}, + ] + ) + + result = run_calibration_phase1( + ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60 + ) + + assert {(r.kind, r.offset) for r in result.regimes} == { + ("roman", 2), + ("decimal", 5), + } + assert fake.calls == [("Preface", 4), ("Chapter 1", 4)] + + +def test_confirmed_anchor_is_reported_as_a_sample(patch_scan) -> None: + patch_scan(_FakeScan({"Chapter 1": 15})) + hierarchies = _hierarchy( + [{"heading": "Chapter 1", "page_number": "10", "level": 1}] + ) + + result = run_calibration_phase1( + ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60 + ) + + sample = result.regimes[0].samples[0] + assert (sample.title, sample.physical) == ("Chapter 1", 15) + + +def test_entries_without_a_parseable_printed_page_are_skipped(patch_scan) -> None: + fake = patch_scan(_FakeScan({"Chapter 1": 15})) + hierarchies = _hierarchy( + [ + {"heading": "Cover", "page_number": None, "level": 1}, + {"heading": "Chapter 1", "page_number": "10", "level": 1}, + ] + ) + + run_calibration_phase1(ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60) + + assert fake.calls == [("Chapter 1", 4)] + + +def test_empty_toc_fails_without_scanning(patch_scan) -> None: + fake = patch_scan(_FakeScan({})) + + result = run_calibration_phase1(ctx=_ctx(), toc_hierarchies=[], page_count=60) + + assert result.failure_kind == FAILURE_TOC_EMPTY + assert fake.calls == [] + + +def test_missing_page_count_fails_without_scanning(patch_scan) -> None: + fake = patch_scan(_FakeScan({})) + hierarchies = _hierarchy( + [{"heading": "Chapter 1", "page_number": "10", "level": 1}] + ) + + result = run_calibration_phase1( + ctx=_ctx(page_count=0), toc_hierarchies=hierarchies, page_count=0 + ) + + assert result.failure_kind == FAILURE_PAGE_COUNT_MISSING + assert fake.calls == [] diff --git a/apps/worker/tests/contract/test_calibration_scan_contract.py b/apps/worker/tests/contract/test_calibration_scan_contract.py new file mode 100644 index 000000000..83191311a --- /dev/null +++ b/apps/worker/tests/contract/test_calibration_scan_contract.py @@ -0,0 +1,213 @@ +"""Contract tests for the deterministic forward title scan (calibration Phase-1).""" + +from __future__ import annotations + +import os +from typing import Any + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +import pytest + +from app.services.document_agent.calibration import scan as scan_module +from app.services.document_agent.calibration.scan import ( + DEFAULT_WINDOW_SCHEDULE, + scan_title_forward, +) +from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.state import ProfileBlackboard + + +def _ctx(page_count: int = 60) -> ToolContext: + blackboard = ProfileBlackboard() + blackboard.page_count = page_count + return ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="job-scan", + blackboard=blackboard, + trace=None, + settings={"vlm_model": "test-vlm"}, + ) + + +class _FakeInspect: + """Records every inspect call and answers hit only on ``hit_page``.""" + + def __init__(self, hit_page: int | None) -> None: + self.hit_page = hit_page + self.calls: list[list[int]] = [] + + def __call__(self, ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + pages = list(args.get("pages") or []) + self.calls.append(pages) + hit = self.hit_page in pages if self.hit_page is not None else False + return ToolResult( + status="ok", + payload={ + "pages": pages, + "answer": "", + "fields": { + "found": hit, + "found_page": self.hit_page if hit else None, + }, + }, + ) + + +@pytest.fixture +def patch_inspect(monkeypatch: pytest.MonkeyPatch): + def _apply(fake: _FakeInspect) -> _FakeInspect: + monkeypatch.setattr(scan_module, "inspect_pages", fake) + return fake + + return _apply + + +def test_first_round_opens_the_candidate_page_and_its_successor(patch_inspect) -> None: + fake = patch_inspect(_FakeInspect(hit_page=10)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert fake.calls == [[10, 11]] + assert result.found is True + assert result.found_page == 10 + + +def test_miss_expands_forward_from_the_cursor_without_rescanning(patch_inspect) -> None: + fake = patch_inspect(_FakeInspect(hit_page=None)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert fake.calls == [ + [10, 11], + [12, 13, 14, 15], + [16, 17, 18, 19, 20, 21], + [22, 23, 24, 25, 26, 27, 28, 29, 30, 31], + ] + assert result.found is False + assert result.scanned_pages == list(range(10, 32)) + assert len(result.scanned_pages) == len(set(result.scanned_pages)) + assert result.next_start == 32 + + +def test_scan_stops_at_first_hit(patch_inspect) -> None: + fake = patch_inspect(_FakeInspect(hit_page=13)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert fake.calls == [[10, 11], [12, 13, 14, 15]] + assert result.found_page == 13 + assert result.next_start == 16 + + +def test_scan_covers_at_most_the_window_schedule(patch_inspect) -> None: + fake = patch_inspect(_FakeInspect(hit_page=None)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert len(fake.calls) == len(DEFAULT_WINDOW_SCHEDULE) + assert len(result.scanned_pages) == sum(DEFAULT_WINDOW_SCHEDULE) + + +def test_window_is_clipped_at_the_last_page(patch_inspect) -> None: + fake = patch_inspect(_FakeInspect(hit_page=None)) + + result = scan_title_forward( + ctx=_ctx(page_count=13), title="Appendix", start_page=10, page_count=13 + ) + + assert fake.calls == [[10, 11], [12, 13]] + assert result.next_start is None + + +def test_each_call_lifts_the_page_cap_to_its_own_window(patch_inspect) -> None: + class _Recording(_FakeInspect): + def __init__(self) -> None: + super().__init__(hit_page=None) + self.caps: list[int] = [] + + def __call__(self, ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + self.caps.append(int(args["page_cap"])) + return super().__call__(ctx, args) + + fake = patch_inspect(_Recording()) + + scan_title_forward(ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60) + + assert fake.caps == list(DEFAULT_WINDOW_SCHEDULE) + + +def test_inspect_error_aborts_the_scan(patch_inspect) -> None: + class _Failing(_FakeInspect): + def __call__(self, ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + self.calls.append(list(args.get("pages") or [])) + return ToolResult(status="error", error="calibration visual budget exhausted") + + fake = patch_inspect(_Failing(hit_page=None)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert fake.calls == [[10, 11]] + assert result.found is False + assert result.rounds[-1].error == "calibration visual budget exhausted" + + +def test_string_false_is_not_treated_as_a_hit(patch_inspect) -> None: + class _StringBool(_FakeInspect): + def __call__(self, ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + pages = list(args.get("pages") or []) + self.calls.append(pages) + return ToolResult( + status="ok", + payload={ + "fields": { + "found": "false", + "found_page": pages[0] if pages else None, + } + }, + ) + + fake = patch_inspect(_StringBool(hit_page=None)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert result.found is False + assert result.found_page is None + assert len(fake.calls) == len(DEFAULT_WINDOW_SCHEDULE) + + +def test_found_page_outside_the_window_is_rejected(patch_inspect) -> None: + class _Liar(_FakeInspect): + def __call__(self, ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + pages = list(args.get("pages") or []) + self.calls.append(pages) + return ToolResult( + status="ok", + payload={"fields": {"found": True, "found_page": 999}}, + ) + + patch_inspect(_Liar(hit_page=None)) + + result = scan_title_forward( + ctx=_ctx(), title="Chapter 1", start_page=10, page_count=60 + ) + + assert result.found is False + assert result.found_page is None diff --git a/apps/worker/tests/contract/test_coarse_profile_protocol_contract.py b/apps/worker/tests/contract/test_coarse_profile_protocol_contract.py new file mode 100644 index 000000000..9adc84731 --- /dev/null +++ b/apps/worker/tests/contract/test_coarse_profile_protocol_contract.py @@ -0,0 +1,158 @@ +"""Protocol tests: coarse profile parse + deterministic shard finalize.""" + +from __future__ import annotations + +import json +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_agent.coarse_profile.classifier import _parse_profile +from app.services.document_agent.coordinator import ProfileCoordinator +from app.services.document_agent.manifest import ( + DocumentProfile, + PageFeature, + PageLabel, + TocResult, +) + + +def test_parse_profile_reads_classification_fields() -> None: + raw = json.dumps( + { + "is_scanned": True, + "category": "Feasibility Study Report", + "routing_category": "generic", + "language": "zh", + "rationale": "scanned PDF not atlas", + "header_y": None, + "footer_y": None, + } + ) + profile = _parse_profile(raw) + assert profile.is_scanned is True + assert profile.category == "Feasibility Study Report" + assert profile.routing_category == "generic" + assert profile.language == "zh" + + +def test_parse_profile_rejects_invalid_header_footer_order() -> None: + raw = json.dumps( + { + "is_scanned": False, + "category": "Report", + "routing_category": "generic", + "language": "en", + "rationale": "ok", + "header_y": 0.8, + "footer_y": 0.2, + } + ) + profile = _parse_profile(raw) + assert profile.header_y is None + assert profile.footer_y is None + + +def _seed_pages(coordinator: ProfileCoordinator, page_count: int) -> None: + blackboard = coordinator.blackboard + blackboard.page_count = page_count + blackboard.doc_stats = {"page_count": page_count} + blackboard.page_features = [ + PageFeature( + page=page, + raw_text_length=0, + text_density=0.0, + image_coverage=1.0, + image_count=1, + table_count=0, + drawings_count=0, + orientation="portrait", + width=612.0, + height=792.0, + has_asset=True, + is_blank_like=True, + ) + for page in range(1, page_count + 1) + ] + blackboard.page_labels = [ + PageLabel(page=page, kind="normal") + for page in range(1, page_count + 1) + ] + blackboard.toc_result = TocResult(method="none", notes="no toc") + blackboard.document_profile = DocumentProfile( + is_scanned=True, + category="Report", + routing_category="generic", + rationale="fixture", + ) + + +def test_finalize_shard_plan_reaches_success(tmp_path) -> None: + coordinator = ProfileCoordinator( + pdf_path=str(tmp_path / "doc.pdf"), + job_id="job-finalize", + output_dir=str(tmp_path / "out"), + ) + (tmp_path / "out").mkdir() + _seed_pages(coordinator, 4) + + coordinator._finalize_shard_plan() + + assert coordinator.blackboard.verdict is not None + assert coordinator.blackboard.verdict.status == "success" + assert coordinator.blackboard.shard_plan is not None + assert len(coordinator.blackboard.shard_plan.shards) >= 1 + + +def test_finalize_shard_plan_aborts_when_validation_fails(tmp_path) -> None: + from app.services.document_agent.manifest import ShardPlan + + coordinator = ProfileCoordinator( + pdf_path=str(tmp_path / "doc.pdf"), + job_id="job-finalize-abort", + output_dir=str(tmp_path / "out"), + ) + (tmp_path / "out").mkdir() + _seed_pages(coordinator, 4) + coordinator.blackboard.shard_plan = ShardPlan( + enabled=True, + reason="too_large", + shards=[], + ) + coordinator.blackboard.validation_report = { + "valid": False, + "errors": ["shard_plan has no shards"], + "warnings": [], + } + + try: + coordinator._finalize_shard_plan() + raise AssertionError("expected shard validation failure to abort") + except RuntimeError as exc: + assert "Shard plan validation failed" in str(exc) + + assert coordinator.blackboard.verdict is not None + assert coordinator.blackboard.verdict.status == "abort" + assert "fallback" not in (coordinator.blackboard.verdict.rationale or "").lower() + + +def test_finalize_shard_plan_creates_plan_when_missing(tmp_path) -> None: + coordinator = ProfileCoordinator( + pdf_path=str(tmp_path / "doc.pdf"), + job_id="job-finalize-missing", + output_dir=str(tmp_path / "out"), + ) + (tmp_path / "out").mkdir() + _seed_pages(coordinator, 3) + assert coordinator.blackboard.shard_plan is None + + coordinator._finalize_shard_plan() + + assert coordinator.blackboard.shard_plan is not None + assert len(coordinator.blackboard.shard_plan.shards) == 1 + assert coordinator.blackboard.verdict.status == "success" diff --git a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py index 5a0d8b49e..a8c27f2dc 100644 --- a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py +++ b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py @@ -18,7 +18,6 @@ from app.services.document_agent.trace import ParseRunRecorder from app.services.document_agent.manifest import ( DocumentProfile, - H1BoundaryResult, PageAnatomyMap, PageFeature, PageLabel, @@ -76,7 +75,7 @@ def _seed_preprobed_pages( coordinator.blackboard.page_count = page_count coordinator.blackboard.page_features = [_page_feature(page) for page in probed_pages] coordinator.blackboard.page_labels = [ - PageLabel(page=page, kind="normal", confidence=1.0) for page in probed_pages + PageLabel(page=page, kind="normal") for page in probed_pages ] coordinator.blackboard.doc_stats = {"page_count": page_count} coordinator.blackboard.global_signals["page_kind_counts"] = { @@ -85,28 +84,32 @@ def _seed_preprobed_pages( coordinator.blackboard.global_signals["assets_probed"] = True -def test_toc_anchor_text_scan_matches_full_page_and_cross_line_keywords() -> None: +def test_toc_anchor_text_scan_whole_line_keyword_and_split_repair() -> None: late_lines = [f"body line {idx}" for idx in range(60)] + ["目录"] split_lines = ["Table of", "Con", "tents"] + false_positive_lines = [ + "Commentary provides guidance on minimum cement contents in different situations.", + "The basic contents of a typical contract document are shown below:", + ] - late_matches = toc_anchor_tool._find_toc_text_matches( # noqa: SLF001 - late_lines, - cross_line_window=6, - ) - split_matches = toc_anchor_tool._find_toc_text_matches( # noqa: SLF001 - split_lines, - cross_line_window=6, + late_matches = toc_anchor_tool._find_toc_text_matches(late_lines) # noqa: SLF001 + split_matches = toc_anchor_tool._find_toc_text_matches(split_lines) # noqa: SLF001 + false_matches = toc_anchor_tool._find_toc_text_matches( # noqa: SLF001 + false_positive_lines ) assert late_matches[0]["line_index"] == 60 assert late_matches[0]["match_kind"] == "keyword:目录" - assert split_matches[0]["match_kind"] == "cross_line:tableofcontents" + assert split_matches[0]["match_kind"] == "keyword:tableofcontents" + assert split_matches[0]["line_index"] == 0 + assert split_matches[0]["line_end_index"] == 2 + assert false_matches == [] -def test_toc_extraction_degrades_to_empty_result_on_failure(tmp_path: Path) -> None: +def test_toc_extraction_raises_on_pipeline_failure(tmp_path: Path) -> None: coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "standard.pdf"), - job_id="job-toc-fail-soft", + job_id="job-toc-fail-hard", output_dir=str(tmp_path / "profile"), ) coordinator.blackboard.page_count = 1 @@ -117,18 +120,17 @@ def _fail_toc_extraction() -> None: coordinator._run_toc_extraction_pipeline = _fail_toc_extraction # type: ignore[method-assign] - coordinator._ensure_toc_profile(strict=False) - toc_result = coordinator.blackboard.toc_result + try: + coordinator._ensure_toc_profile(strict=False) + raise AssertionError("expected TOC pipeline failure to raise") + except RuntimeError as exc: + assert "VLM JSON parse failed" in str(exc) - assert toc_result is not None - assert toc_result.method == "none" - assert toc_result.toc_pages == [] - assert toc_result.failure_kind == "degraded" - assert "degraded" in toc_result.notes + assert coordinator.blackboard.toc_result is None assert coordinator.blackboard.toc_hierarchies is None -def test_run_lightweight_anatomy_builds_single_shard_without_planner_llm( +def test_run_lightweight_anatomy_builds_single_shard_without_coarse_vlm( tmp_path: Path, ) -> None: output_dir = tmp_path / "profile" @@ -162,10 +164,10 @@ def test_run_lightweight_anatomy_builds_single_shard_without_planner_llm( assert "text_lines_preview" not in anatomy_data["page_features"][0] assert "asset_bboxes" not in anatomy_data["page_features"][0] trace_data = json.loads((output_dir / "trace.json").read_text(encoding="utf-8")) - assert "visual_stages" in trace_data["summary"]["budget"] + assert "budget" not in trace_data["summary"] -def test_run_coarse_runs_asset_probe_after_planner(monkeypatch, tmp_path: Path) -> None: +def test_run_coarse_runs_asset_probe_after_coarse_profile(monkeypatch, tmp_path: Path) -> None: coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "doc.pdf"), job_id="job-asset-probe-after-coarse", @@ -175,23 +177,22 @@ def test_run_coarse_runs_asset_probe_after_planner(monkeypatch, tmp_path: Path) coordinator.blackboard.page_count = 2 coordinator.blackboard.page_features = [_page_feature(1), _page_feature(2)] coordinator.blackboard.page_labels = [ - PageLabel(page=1, kind="normal", confidence=1.0), - PageLabel(page=2, kind="normal", confidence=1.0), + PageLabel(page=1, kind="normal"), + PageLabel(page=2, kind="normal"), ] coordinator.blackboard.doc_stats = {"page_count": 2} coordinator.blackboard.global_signals["page_kind_counts"] = {"normal": 2} calls: list[str] = [] - def fake_propose(_self): - calls.append("planner") + def fake_classify(_self): + calls.append("coarse_profile") return ( DocumentProfile( is_scanned=False, category="Research Report", routing_category=PdfRoutingCategory.GENERIC.value, ), - None, ToolResult(status="ok", payload={}), ) @@ -212,7 +213,7 @@ def fake_toc() -> None: calls.append("toc") coordinator.blackboard.toc_result = TocResult(method="none") - monkeypatch.setattr(coordinator_module.ProfilePlanner, "propose", fake_propose) + monkeypatch.setattr(coordinator_module.CoarseProfiler, "classify", fake_classify) monkeypatch.setattr(coordinator_module, "probe_page_assets", fake_probe_page_assets) monkeypatch.setattr(coordinator_module, "aggregate_doc_stats", fake_aggregate) monkeypatch.setattr(coordinator, "_run_text_scan", fake_text_scan) @@ -222,7 +223,7 @@ def fake_toc() -> None: assert profile.category == "Research Report" assert calls == [ - "planner", + "coarse_profile", "text_scan", "probe.page_assets", "aggregate.doc_stats", @@ -247,11 +248,10 @@ def test_parse_run_recorder_doc_profile_uses_final_anatomy_toc() -> None: page_count=2, page_features=[_page_feature(1), _page_feature(2)], page_labels=[ - PageLabel(page=1, kind="normal", confidence=1.0), - PageLabel(page=2, kind="normal", confidence=1.0), + PageLabel(page=1, kind="normal"), + PageLabel(page=2, kind="normal"), ], toc_result=TocResult(toc_pages=[2], method="vlm_batch", notes="ok"), - h1_result=H1BoundaryResult(method="toc_grep"), shard_plan=ShardPlan(enabled=False, reason="not_needed"), toc_hierarchies=[{"toc_range": [2, 2], "toc_tree": {}}], global_signals={"toc_profile_attempted": True}, @@ -302,9 +302,8 @@ def rollback(self) -> None: file_path="/tmp/doc.pdf", page_count=12, page_features=[_page_feature(1)], - page_labels=[PageLabel(page=1, kind="normal", confidence=1.0)], + page_labels=[PageLabel(page=1, kind="normal")], toc_result=TocResult(toc_pages=[2], method="vlm_batch", notes="ok"), - h1_result=H1BoundaryResult(method="toc_grep"), shard_plan=ShardPlan(enabled=False, reason="not_needed"), toc_hierarchies=[{"toc_range": [2, 2], "toc_tree": {}}], global_signals={"toc_profile_attempted": True}, @@ -340,7 +339,6 @@ def test_run_structural_retries_transient_confirm_failed_toc_result( AgentTocEvidence( page_index=17, source="vlm", - confidence=0.05, reason="rejected", ) ], @@ -365,42 +363,36 @@ def fake_persist(_anatomy): monkeypatch.setattr(coordinator, "_persist_ready_anatomy", fake_persist) monkeypatch.setattr( - coordinator_module.ProfilePlanner, - "propose", + coordinator_module.CoarseProfiler, + "classify", lambda self: ( coordinator.blackboard.document_profile, - None, ToolResult(status="ok", payload={}), ), ) - class FakeExecutor: - def __init__(self, *_args, **_kwargs) -> None: - pass + def fake_finalize() -> None: + coordinator.blackboard.shard_plan = ShardPlan( + enabled=True, + reason="too_large", + shards=[ + Shard( + shard_index=0, + page_start=1, + page_end=3, + page_offset=0, + anchor_type="forced_max_size", + anchor_evidence="fixture", + ) + ], + ) + from app.services.document_agent.manifest import ProfileVerdict - def run(self): - coordinator.blackboard.shard_plan = ShardPlan( - enabled=True, - reason="too_large", - shards=[ - Shard( - shard_index=0, - page_start=1, - page_end=3, - page_offset=0, - anchor_type="forced_max_size", - anchor_evidence="fixture", - confidence=1.0, - ) - ], - ) - return SimpleNamespace( - success=True, - verdict=SimpleNamespace(status="success", rationale="ok"), - trace_summary={}, - ) + coordinator.blackboard.verdict = ProfileVerdict( + status="success", rationale="ok" + ) - monkeypatch.setattr(coordinator_module, "ReActExecutor", FakeExecutor) + monkeypatch.setattr(coordinator, "_finalize_shard_plan", fake_finalize) anatomy = coordinator.run_structural() @@ -408,7 +400,7 @@ def run(self): assert anatomy.toc_result.toc_pages == [17] -def test_run_structural_skip_shard_plan_uses_placeholder_without_executor( +def test_run_structural_skip_shard_plan_uses_placeholder_without_finalize( monkeypatch, tmp_path: Path, ) -> None: @@ -446,23 +438,21 @@ def test_run_structural_skip_shard_plan_uses_placeholder_without_executor( lambda _anatomy: None, ) monkeypatch.setattr( - coordinator_module.ProfilePlanner, - "propose", + coordinator_module.CoarseProfiler, + "classify", lambda self: ( coordinator.blackboard.document_profile, - None, ToolResult(status="ok", payload={}), ), ) - class BoomExecutor: - def __init__(self, *_args, **_kwargs) -> None: - raise AssertionError("ReActExecutor must not run when skip_shard_plan") - - def run(self): # pragma: no cover - raise AssertionError("unreachable") + class BoomFinalize: + def __call__(self) -> None: + raise AssertionError( + "_finalize_shard_plan must not run when skip_shard_plan" + ) - monkeypatch.setattr(coordinator_module, "ReActExecutor", BoomExecutor) + monkeypatch.setattr(coordinator, "_finalize_shard_plan", BoomFinalize()) anatomy = coordinator.run_structural(skip_shard_plan=True) @@ -514,42 +504,36 @@ def fake_persist(_anatomy): monkeypatch.setattr(coordinator, "_persist_ready_anatomy", fake_persist) monkeypatch.setattr( - coordinator_module.ProfilePlanner, - "propose", + coordinator_module.CoarseProfiler, + "classify", lambda self: ( coordinator.blackboard.document_profile, - None, ToolResult(status="ok", payload={}), ), ) - class FakeExecutor: - def __init__(self, *_args, **_kwargs) -> None: - pass + def fake_finalize() -> None: + coordinator.blackboard.shard_plan = ShardPlan( + enabled=True, + reason="too_large", + shards=[ + Shard( + shard_index=0, + page_start=1, + page_end=3, + page_offset=0, + anchor_type="forced_max_size", + anchor_evidence="fixture", + ) + ], + ) + from app.services.document_agent.manifest import ProfileVerdict - def run(self): - coordinator.blackboard.shard_plan = ShardPlan( - enabled=True, - reason="too_large", - shards=[ - Shard( - shard_index=0, - page_start=1, - page_end=3, - page_offset=0, - anchor_type="forced_max_size", - anchor_evidence="fixture", - confidence=1.0, - ) - ], - ) - return SimpleNamespace( - success=True, - verdict=SimpleNamespace(status="success", rationale="ok"), - trace_summary={}, - ) + coordinator.blackboard.verdict = ProfileVerdict( + status="success", rationale="ok" + ) - monkeypatch.setattr(coordinator_module, "ReActExecutor", FakeExecutor) + monkeypatch.setattr(coordinator, "_finalize_shard_plan", fake_finalize) anatomy = coordinator.run_structural() @@ -558,13 +542,13 @@ def run(self): assert anatomy.toc_result.toc_pages == [] -def test_run_coarse_runs_planner_then_text_scan_then_toc( +def test_run_coarse_runs_coarse_profile_then_text_scan_then_toc( monkeypatch, tmp_path: Path, ) -> None: coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "oversized.pdf"), - job_id="job-planner-then-toc", + job_id="job-coarse-then-toc", output_dir=str(tmp_path / "profile"), settings={"toc_profile_enabled": True}, ) @@ -591,52 +575,46 @@ def fake_persist(_anatomy): monkeypatch.setattr(coordinator, "_run_text_scan", fake_text_scan) monkeypatch.setattr(coordinator, "_persist_ready_anatomy", fake_persist) - def fake_propose(_self): - calls.append("planner") + def fake_classify(_self): + calls.append("coarse_profile") return ( DocumentProfile( is_scanned=False, category="Prospectus", routing_category=PdfRoutingCategory.GENERIC.value, ), - None, ToolResult(status="ok", payload={}), ) - monkeypatch.setattr(coordinator_module.ProfilePlanner, "propose", fake_propose) + monkeypatch.setattr(coordinator_module.CoarseProfiler, "classify", fake_classify) - class FakeExecutor: - def __init__(self, *_args, **_kwargs) -> None: - pass + def fake_finalize() -> None: + coordinator.blackboard.shard_plan = ShardPlan( + enabled=True, + reason="too_large", + shards=[ + Shard( + shard_index=0, + page_start=1, + page_end=3, + page_offset=0, + anchor_type="forced_max_size", + anchor_evidence="fixture", + ) + ], + ) + from app.services.document_agent.manifest import ProfileVerdict - def run(self): - coordinator.blackboard.shard_plan = ShardPlan( - enabled=True, - reason="too_large", - shards=[ - Shard( - shard_index=0, - page_start=1, - page_end=3, - page_offset=0, - anchor_type="forced_max_size", - anchor_evidence="fixture", - confidence=1.0, - ) - ], - ) - return SimpleNamespace( - success=True, - verdict=SimpleNamespace(status="success", rationale="ok"), - trace_summary={}, - ) + coordinator.blackboard.verdict = ProfileVerdict( + status="success", rationale="ok" + ) - monkeypatch.setattr(coordinator_module, "ReActExecutor", FakeExecutor) + monkeypatch.setattr(coordinator, "_finalize_shard_plan", fake_finalize) coordinator.run_coarse() anatomy = coordinator.run_structural() - assert calls == ["planner", "text_scan", "toc", "persist"] + assert calls == ["coarse_profile", "text_scan", "toc", "persist"] assert anatomy.toc_result.toc_pages == [17] @@ -727,7 +705,6 @@ def test_oversized_single_shard_plan_is_invalid() -> None: page_offset=0, anchor_type="forced_max_size", anchor_evidence="final shard", - confidence=1.0, ) ], ), @@ -884,7 +861,7 @@ def run_lightweight_anatomy(self, *, skip_shard_plan: bool = False): monkeypatch.setattr(doc_profiler, "ProfileCoordinator", FakeCoordinator) monkeypatch.setattr(doc_profiler.settings, "MAX_PDF_PAGE_LIMIT", 200) - # Global kill switch OFF; page_memory track must still enable TOC. + # Kill switch OFF for chunk; page_memory must still force TOC on. monkeypatch.setattr(doc_profiler.settings, "PDF_PROFILE_TOC_ENABLED", False) profile = profile_document( @@ -978,7 +955,6 @@ def run_coarse(self) -> DocumentProfile: AgentTocEvidence( page_index=2, source="vlm", - confidence=0.95, reason="table of contents", ) ], @@ -1021,7 +997,7 @@ def __init__(self, **kwargs) -> None: assert profile.toc.has_toc is True assert profile.toc.attempted is True assert profile.toc.method == "vlm_batch" - assert profile.toc.evidence[0].confidence == 0.95 + assert profile.toc.evidence[0].source == "vlm" assert profile.anatomy is fake_anatomy @@ -1050,7 +1026,6 @@ def run_coarse(self) -> DocumentProfile: AgentTocEvidence( page_index=4, source="vlm", - confidence=0.9, reason="table of contents", ) ], @@ -1270,11 +1245,10 @@ def fake_parse_md(*_args, **kwargs): page_count=2, page_features=[_page_feature(1), _page_feature(2)], page_labels=[ - PageLabel(page=1, kind="normal", confidence=1.0), - PageLabel(page=2, kind="normal", confidence=1.0), + PageLabel(page=1, kind="normal"), + PageLabel(page=2, kind="normal"), ], toc_result=TocResult(method="none"), - h1_result=H1BoundaryResult(method="none"), shard_plan=ShardPlan( enabled=False, reason="not_needed", @@ -1286,7 +1260,6 @@ def fake_parse_md(*_args, **kwargs): page_offset=0, anchor_type="forced_max_size", anchor_evidence="document within shard threshold", - confidence=1.0, ) ], ), @@ -1357,12 +1330,11 @@ def fake_eval_md_headings(md_lines, *_args, **kwargs): page_count=3, page_features=[_page_feature(1), _page_feature(2), _page_feature(3)], page_labels=[ - PageLabel(page=1, kind="normal", confidence=1.0), - PageLabel(page=2, kind="normal", confidence=1.0), - PageLabel(page=3, kind="normal", confidence=1.0), + PageLabel(page=1, kind="normal"), + PageLabel(page=2, kind="normal"), + PageLabel(page=3, kind="normal"), ], toc_result=TocResult(method="none"), - h1_result=H1BoundaryResult(method="none"), shard_plan=ShardPlan( enabled=False, reason="not_needed", @@ -1374,7 +1346,6 @@ def fake_eval_md_headings(md_lines, *_args, **kwargs): page_offset=0, anchor_type="forced_max_size", anchor_evidence="document within shard threshold", - confidence=1.0, ) ], ), @@ -1399,11 +1370,17 @@ def fake_eval_md_headings(md_lines, *_args, **kwargs): def test_page_based_toc_demotes_front_matter_only_on_first_shard() -> None: + # Production TEXT-TRACK shape from propose.shard_plan: calibrated slice + # carries toc_with_level only (no toc_tree). toc_hierarchies = [ { - "toc_range": [2, 2], + "toc_range": [1, 10], "toc_range_unit": "page", - "toc_tree": {"Risk Factors": {}}, + "source": "calibrated_shard_split", + "toc_with_level": [ + {"heading": "Risk Factors", "level": 1}, + {"heading": "Business Overview", "level": 2}, + ], } ] lines = [ diff --git a/apps/worker/tests/contract/test_ocr_pages_contract.py b/apps/worker/tests/contract/test_ocr_pages_contract.py index d47497283..d55254046 100644 --- a/apps/worker/tests/contract/test_ocr_pages_contract.py +++ b/apps/worker/tests/contract/test_ocr_pages_contract.py @@ -14,14 +14,13 @@ os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") os.environ.setdefault("S3_TEMP_PATH", "/tmp") -from app.services.document_agent.budget import BudgetTracker from app.services.document_agent.manifest import PageFeature, ToolContext -from app.services.document_agent.state import AgentBlackboard +from app.services.document_agent.state import ProfileBlackboard from app.services.document_agent.tools.ocr_pages import ocr_pages def _ctx() -> ToolContext: - blackboard = AgentBlackboard(page_count=2) + blackboard = ProfileBlackboard(page_count=2) blackboard.page_features = [ PageFeature( page=1, @@ -42,7 +41,6 @@ def _ctx() -> ToolContext: pdf_path="/tmp/doc.pdf", job_id="job-ocr", blackboard=blackboard, - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), trace=None, settings={}, ) diff --git a/apps/worker/tests/contract/test_outline_check_contract.py b/apps/worker/tests/contract/test_outline_check_contract.py new file mode 100644 index 000000000..6a3864268 --- /dev/null +++ b/apps/worker/tests/contract/test_outline_check_contract.py @@ -0,0 +1,64 @@ +"""Contract tests for outline_check self-verify and digest helpers.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_agent.structure.outline_check import ( + build_tree_digest_from_entries, + flatten_outline_entries, + verify_entries, +) + + +def test_verify_entries_drops_mismatched_page_title() -> None: + entries = [ + {"heading": "Keep Me", "level": 1, "page": 2}, + {"heading": "Missing", "level": 1, "page": 3}, + {"heading": "No Page Parent", "level": 1, "page": None}, + ] + page_texts = { + 2: "Preface\nKeep Me\nBody", + 3: "Other chapter text only", + } + kept, dropped = verify_entries(entries, page_texts) + assert [row["heading"] for row in kept] == ["Keep Me", "No Page Parent"] + assert [row["heading"] for row in dropped] == ["Missing"] + + +def test_verify_entries_zero_paged_alive_means_empty_paged_kept() -> None: + entries = [ + {"heading": "Gone", "level": 1, "page": 1}, + {"heading": "Also Gone", "level": 2, "page": 2}, + ] + kept, dropped = verify_entries(entries, {1: "x", 2: "y"}) + assert kept == [] + assert len(dropped) == 2 + + +def test_flatten_and_digest_preserve_levels() -> None: + roots = [ + { + "title": "Part A", + "level": 1, + "page": None, + "children": [ + {"title": "Chapter 1", "level": 2, "page": 10, "children": []}, + ], + } + ] + entries = flatten_outline_entries(roots) + assert entries == [ + {"heading": "Part A", "level": 1, "page": None}, + {"heading": "Chapter 1", "level": 2, "page": 10}, + ] + digest = build_tree_digest_from_entries(entries) + assert "L1 Part A" in digest + assert "L2 Chapter 1" in digest diff --git a/apps/worker/tests/contract/test_outline_short_circuit_contract.py b/apps/worker/tests/contract/test_outline_short_circuit_contract.py new file mode 100644 index 000000000..cb038742f --- /dev/null +++ b/apps/worker/tests/contract/test_outline_short_circuit_contract.py @@ -0,0 +1,233 @@ +"""OUTLINE route inside run_toc_anchoring: skip calibrate VLM when outline wins.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest.mock import patch + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_agent.manifest import TocResult, ToolContext, ToolResult +from app.services.document_agent.state import ProfileBlackboard +from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring + + +def _outline_roots() -> list[dict[str, Any]]: + return [ + { + "title": "第一章", + "level": 1, + "page": 10, + "children": [ + { + "title": "第二章", + "level": 2, + "page": 15, + "children": [], + } + ], + } + ] + + +def _ctx(*, toc_pages: list[int] | None = None) -> ToolContext: + pages = [2, 3, 4, 5] if toc_pages is None else toc_pages + blackboard = ProfileBlackboard(page_count=20) + blackboard.page_full_text_cache = { + 2: "目录\n第一章 ...... 3\n第二章 ...... 8", + 3: "目录续\n更多条目", + 4: "目录续\n再多条目", + 5: "目录尾", + 10: "第一章\n正文", + 15: "第二章\n正文", + } + # Written by find-stage probe.outline; anchoring only consumes it. + blackboard.pdf_outline_roots = _outline_roots() + blackboard.toc_result = TocResult( + method="vlm_batch", + toc_pages=pages, + notes="confirmed toc pages from extract", + ) + blackboard.toc_hierarchies = [ + { + "source": "vlm", + "toc_range": [2, 5], + "toc_range_unit": "page", + "toc_with_level": [ + {"heading": "第一章", "level": 1, "page_number": 3}, + {"heading": "第二章", "level": 1, "page_number": 8}, + ], + } + ] + return ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="job-outline-anchor", + blackboard=blackboard, + trace=None, + settings={}, + ) + + +def _no_null_parent_locate(**kwargs: Any) -> tuple[dict[Any, Any], list[Any]]: + return dict(kwargs["match_overrides"]), [] + + +def test_outline_wins_skips_calibrate_and_keeps_confirmed_toc_pages() -> None: + ctx = _ctx() + calibrate_calls = {"count": 0} + judge_calls: list[dict[str, Any]] = [] + probe_calls = {"count": 0} + + def fake_judge(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + del tool_ctx + judge_calls.append(args) + return ToolResult( + status="ok", + payload={"choice": "outline", "reason": "broader coverage"}, + ) + + def fake_probe(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + del tool_ctx, args + probe_calls["count"] += 1 + return ToolResult(status="ok", payload={"roots": []}) + + def fake_calibrate(*args: Any, **kwargs: Any) -> Any: + del args, kwargs + calibrate_calls["count"] += 1 + raise AssertionError("calibrate_offset must not run when outline wins") + + with ( + patch( + "app.services.document_agent.tools.probe_outline.probe_outline", + side_effect=fake_probe, + ), + patch( + "app.services.document_agent.tools.judge_toc_source.judge_toc_source", + side_effect=fake_judge, + ), + patch( + "app.services.document_agent.calibration.service.calibrate_offset", + side_effect=fake_calibrate, + ), + patch( + "app.services.document_agent.structure.anchoring_primitives." + "locate_null_page_parent_overrides", + side_effect=_no_null_parent_locate, + ), + ): + run_toc_anchoring(ctx) + + assert probe_calls["count"] == 0 + assert calibrate_calls["count"] == 0 + assert len(judge_calls) == 1 + assert judge_calls[0]["toc_pages"] == [2, 3, 4, 5] + assert ctx.blackboard.toc_result is not None + assert ctx.blackboard.toc_result.toc_pages == [2, 3, 4, 5] + assert ctx.blackboard.toc_result.method == "pdf_outline" + assert ctx.blackboard.skeleton_anchor is not None + assert ctx.blackboard.skeleton_anchor["source"] == "pdf_outline" + assert ctx.blackboard.skeleton_anchor["offset_status"] == "ok" + assert ctx.blackboard.toc_hierarchies is not None + assert ctx.blackboard.toc_hierarchies[0]["source"] == "pdf_outline" + + +def test_outline_without_toc_pages_adopts_without_judge() -> None: + ctx = _ctx(toc_pages=[]) + judge_calls = {"count": 0} + + def fake_judge(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + del tool_ctx, args + judge_calls["count"] += 1 + return ToolResult(status="ok", payload={"choice": "outline"}) + + with ( + patch( + "app.services.document_agent.tools.judge_toc_source.judge_toc_source", + side_effect=fake_judge, + ), + patch( + "app.services.document_agent.structure.anchoring_primitives." + "locate_null_page_parent_overrides", + side_effect=_no_null_parent_locate, + ), + ): + run_toc_anchoring(ctx) + + assert judge_calls["count"] == 0 + assert ctx.blackboard.skeleton_anchor is not None + assert ctx.blackboard.skeleton_anchor["source"] == "pdf_outline" + + +def test_printed_toc_wins_uses_vlm_calibrate_path() -> None: + ctx = _ctx() + vlm_anchor_calls = {"count": 0} + + def fake_judge(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + del tool_ctx, args + return ToolResult( + status="ok", + payload={"choice": "printed_toc", "reason": "printed finer"}, + ) + + def fake_anchor_hierarchy(**kwargs: Any) -> Any: + del kwargs + vlm_anchor_calls["count"] += 1 + from app.services.document_agent.structure.anchoring_primitives import ( + SkeletonAnchor, + ) + + return [], SkeletonAnchor( + offset=0, + offset_status="ok", + match_overrides={}, + null_page_report=[], + bulk_count=0, + ) + + with ( + patch( + "app.services.document_agent.tools.judge_toc_source.judge_toc_source", + side_effect=fake_judge, + ), + patch( + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", + side_effect=fake_anchor_hierarchy, + ), + ): + run_toc_anchoring(ctx) + + assert vlm_anchor_calls["count"] == 1 + assert ctx.blackboard.toc_result is not None + assert ctx.blackboard.toc_result.method == "vlm_batch" + assert ctx.blackboard.toc_hierarchies is not None + assert ctx.blackboard.toc_hierarchies[0]["source"] == "vlm" + + +def test_merge_printed_toc_texts_drops_lines_shared_across_pages() -> None: + from app.services.document_agent.tools.judge_toc_source import ( + merge_printed_toc_texts, + ) + + merged = merge_printed_toc_texts( + [ + "Manual Title\nCONTENTS\nChapter 1 ...... 10", + "Manual Title\nCONTENTS\nChapter 2 ...... 20", + "Manual Title\nChapter 3 ...... 30", + ] + ) + assert merged == "Chapter 1 ...... 10\nChapter 2 ...... 20\nChapter 3 ...... 30" + + +def test_merge_printed_toc_texts_single_page_unchanged() -> None: + from app.services.document_agent.tools.judge_toc_source import ( + merge_printed_toc_texts, + ) + + text = "CONTENTS\nChapter 1 ...... 10\nChapter 1 ...... 10" + assert merge_printed_toc_texts([text]) == text diff --git a/apps/worker/tests/contract/test_page_memory_asset_java_contract.py b/apps/worker/tests/contract/test_page_memory_asset_java_contract.py index 3d809c3d8..3ecfde9dc 100644 --- a/apps/worker/tests/contract/test_page_memory_asset_java_contract.py +++ b/apps/worker/tests/contract/test_page_memory_asset_java_contract.py @@ -65,7 +65,6 @@ def chat_completion_with_usage(self, **kwargs): page=page, source_name="demo.pdf", model_name=page_assets.get_asset_model(), - budget=None, confidence_threshold=0.3, ) @@ -143,7 +142,6 @@ def _fake_extract_table(*, asset, pdf_path, output_dir, table_engine="tabula"): rendered_pages=page_images, output_dir=str(tmp_path), model_name=page_assets.get_asset_model(), - budget=None, max_pages=2, confidence_threshold=0.3, summary_enabled=False, diff --git a/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py b/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py index 3b98debf6..58699fdeb 100644 --- a/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py +++ b/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py @@ -120,7 +120,6 @@ def test_build_node_rows_reuses_tags_without_vlm() -> None: }, filename="demo.pdf", verdict="page", - budget=None, vlm_model=None, ) @@ -166,7 +165,6 @@ def _fake_compute_node_summary(**kwargs): tag_by_page={}, filename="demo.pdf", verdict="page", - budget=None, vlm_model="fake-vlm", node_assembly_concurrency=2, ) @@ -208,7 +206,6 @@ def _fake_compute_node_summary(**kwargs): tag_by_page={}, filename="demo.pdf", verdict="page", - budget=None, vlm_model="fake-vlm", node_assembly_concurrency=1, ) @@ -230,7 +227,6 @@ def test_build_node_rows_attaches_page_citation_assets_for_rendered_pages(tmp_pa }, filename="demo.pdf", verdict="page", - budget=None, vlm_model=None, ) @@ -279,7 +275,6 @@ def test_build_node_rows_keeps_internal_section_body_pages() -> None: }, filename="demo.pdf", verdict="page", - budget=None, vlm_model=None, ) @@ -362,7 +357,6 @@ def chat_completion_with_usage(self, **kwargs): }, filename="demo.pdf", verdict="page", - budget=None, vlm_model="fake-vlm", ) @@ -403,7 +397,6 @@ def test_build_node_rows_prepends_asset_rows_and_links_page_nodes() -> None: }, filename="demo.pdf", verdict="page", - budget=None, vlm_model=None, page_assets_by_page={231: [asset]}, ) diff --git a/apps/worker/tests/contract/test_document_agent_budget_contract.py b/apps/worker/tests/contract/test_page_memory_output_contract.py similarity index 69% rename from apps/worker/tests/contract/test_document_agent_budget_contract.py rename to apps/worker/tests/contract/test_page_memory_output_contract.py index d92909f5d..f86eb7084 100644 --- a/apps/worker/tests/contract/test_document_agent_budget_contract.py +++ b/apps/worker/tests/contract/test_page_memory_output_contract.py @@ -3,6 +3,8 @@ import os from types import SimpleNamespace +import pandas as pd + os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") @@ -10,54 +12,10 @@ os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") os.environ.setdefault("S3_TEMP_PATH", "/tmp") -from app.services.document_agent.budget import BudgetTracker, StageEnvelope from app.services.page_memory import memory_service from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder -import pandas as pd - - -def test_visual_stage_envelope_preserves_other_stage_guarantee() -> None: - budget = BudgetTracker( - plan_budget=100, - visual_budget=100, - visual_stage_envelopes={ - "toc_confirm": StageEnvelope(min_guarantee=30, cap=60), - "coarse_planner": StageEnvelope(min_guarantee=40, cap=70), - }, - ) - - assert budget.try_reserve("visual", 30, stage="toc_confirm") is True - budget.commit("visual", actual=25, est=30, stage="toc_confirm") - assert budget.try_reserve("visual", 36, stage="toc_confirm") is False - - snapshot = budget.snapshot() - assert snapshot["visual"]["used"] == 25 - assert snapshot["visual_stages"]["toc_confirm"]["used"] == 25 - - assert budget.try_reserve("visual", 40, stage="coarse_planner") is True - budget.refund("visual", est=40, stage="coarse_planner") - assert budget.snapshot()["visual_stages"]["coarse_planner"]["reserved"] == 0 - - -def test_visual_stage_cap_rejects_overage_while_legacy_calls_remain_supported() -> None: - budget = BudgetTracker( - plan_budget=100, - visual_budget=100, - visual_stage_envelopes={ - "toc_confirm": StageEnvelope(min_guarantee=0, cap=20), - }, - ) - - assert budget.try_reserve("visual", 21, stage="toc_confirm") is False - assert budget.try_reserve("visual", 90) is True - budget.commit("visual", actual=80, est=90) - - snapshot = budget.snapshot() - assert snapshot["visual"]["used"] == 80 - assert snapshot["visual_stages"]["toc_confirm"]["used"] == 0 - def test_dataframe_converter_accepts_page_chunks_with_extra_metadata() -> None: df = pd.DataFrame( diff --git a/apps/worker/tests/contract/test_parse_task_contract.py b/apps/worker/tests/contract/test_parse_task_contract.py index 398da4e31..9cd005d30 100644 --- a/apps/worker/tests/contract/test_parse_task_contract.py +++ b/apps/worker/tests/contract/test_parse_task_contract.py @@ -895,7 +895,6 @@ def test_oversized_pdf_shard_failure_preserves_processing_error( monkeypatch.setenv("S3_TEMP_PATH", str(tmp_path)) from app.services.document_agent.manifest import ( - H1BoundaryResult, PageAnatomyMap, Shard, ShardPlan, @@ -931,7 +930,6 @@ def _fail_oversized_parse(*args, **kwargs): page_features=[], page_labels=[], toc_result=TocResult(method="none"), - h1_result=H1BoundaryResult(method="none"), shard_plan=ShardPlan( enabled=True, reason="too_large", @@ -943,7 +941,6 @@ def _fail_oversized_parse(*args, **kwargs): page_offset=0, anchor_type="forced_max_size", anchor_evidence="fixture", - confidence=1.0, ) ], ), @@ -968,7 +965,6 @@ def test_oversized_pdf_happy_path_uses_shard_pipeline_without_external_services( monkeypatch.setenv("S3_TEMP_PATH", str(tmp_path)) from app.services.document_agent.manifest import ( - H1BoundaryResult, PageAnatomyMap, Shard, ShardPlan, @@ -999,7 +995,6 @@ def delete_upload_file(self, storage_key: str) -> bool: page_features=[], page_labels=[], toc_result=TocResult(toc_pages=[1], method="vlm_batch"), - h1_result=H1BoundaryResult(method="toc_grep"), shard_plan=ShardPlan( enabled=True, reason="too_large", @@ -1011,7 +1006,6 @@ def delete_upload_file(self, storage_key: str) -> bool: page_offset=0, anchor_type="toc_leaf_boundary", anchor_evidence="Chapter 1", - confidence=0.9, ), Shard( shard_index=1, @@ -1020,7 +1014,6 @@ def delete_upload_file(self, storage_key: str) -> bool: page_offset=2, anchor_type="toc_leaf_boundary", anchor_evidence="Chapter 2", - confidence=0.9, ), ], ), diff --git a/apps/worker/tests/contract/test_probe_links_contract.py b/apps/worker/tests/contract/test_probe_links_contract.py new file mode 100644 index 000000000..2fb1395fc --- /dev/null +++ b/apps/worker/tests/contract/test_probe_links_contract.py @@ -0,0 +1,66 @@ +"""Contract tests for probe.links noise and dest page normalize.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_agent.tools.probe_links import ( + PageLink, + _link_dest_physical_page, + annotate_link_noise, +) + + +def test_link_noise_marks_page_number_header_and_repeated_dest() -> None: + links = [ + PageLink( + source_page=2, + dest_physical_page=10, + anchor_text="12", + from_y0=5, + page_height=100, + ), + PageLink( + source_page=2, + dest_physical_page=10, + anchor_text="Intro", + from_y0=50, + page_height=100, + ), + PageLink( + source_page=3, + dest_physical_page=10, + anchor_text="Intro again", + from_y0=40, + page_height=100, + ), + PageLink( + source_page=1, + dest_physical_page=99, + anchor_text="Normal title", + from_y0=80, + page_height=100, + ), + ] + annotated = annotate_link_noise(links) + by_anchor = {item["anchor_text"]: item for item in annotated} + assert "pure_page_number" in by_anchor["12"]["noise"] + assert "header_zone" in by_anchor["12"]["noise"] + assert "repeated_dest" in by_anchor["12"]["noise"] + assert "repeated_dest" in by_anchor["Intro"]["noise"] + assert by_anchor["Normal title"]["noise"] == [] + + +def test_link_dest_physical_page_by_type() -> None: + """``int`` is 0-based; digit ``str`` is already 1-based.""" + assert _link_dest_physical_page(6) == 7 + assert _link_dest_physical_page(0) == 1 + assert _link_dest_physical_page("6") == 6 + assert _link_dest_physical_page("7") == 7 diff --git a/apps/worker/tests/contract/test_probe_outline_contract.py b/apps/worker/tests/contract/test_probe_outline_contract.py new file mode 100644 index 000000000..a956a932a --- /dev/null +++ b/apps/worker/tests/contract/test_probe_outline_contract.py @@ -0,0 +1,44 @@ +"""Contract tests for probe.outline forest prune and physical pages.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_agent.tools.probe_outline import build_outline_forest + + +def test_outline_keeps_no_page_parent_with_paged_children() -> None: + forest = build_outline_forest( + [ + [1, "Part A", -1], + [2, "Chapter 1", 10], + [2, "Chapter 2", 20], + ] + ) + assert len(forest) == 1 + assert forest[0]["title"] == "Part A" + assert forest[0]["page"] is None + assert [child["title"] for child in forest[0]["children"]] == [ + "Chapter 1", + "Chapter 2", + ] + assert [child["page"] for child in forest[0]["children"]] == [10, 20] + + +def test_outline_drops_no_page_leaf_and_empty_subtree() -> None: + forest = build_outline_forest( + [ + [1, "Keep", 5], + [1, "DropLeaf", -1], + [1, "DropParent", -1], + [2, "DropChild", 0], + ] + ) + assert [node["title"] for node in forest] == ["Keep"] diff --git a/apps/worker/tests/contract/test_profile_agent_protocol_contract.py b/apps/worker/tests/contract/test_profile_agent_protocol_contract.py deleted file mode 100644 index 63452c1ef..000000000 --- a/apps/worker/tests/contract/test_profile_agent_protocol_contract.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Protocol tests: planner next_action and executor finish ownership.""" - -from __future__ import annotations - -import json -import os - -os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") -os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") -os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") -os.environ.setdefault("S3_ACCESS_KEY_ID", "test") -os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") -os.environ.setdefault("S3_TEMP_PATH", "/tmp") - -from app.services.document_agent.budget import BudgetTracker -from app.services.document_agent.executor.react_loop import ( - ReActExecutor, - _parse_decision, -) -from app.services.document_agent.manifest import ( - DocumentProfile, - ReflexionDecision, - ToolContext, -) -from app.services.document_agent.planner.planner import _parse_profile_and_decision -from app.services.document_agent.tools import REGISTRY -from app.services.document_agent.state import AgentBlackboard - - -def test_planner_verdict_now_falls_through_to_ready_to_shard() -> None: - raw = json.dumps( - { - "is_scanned": True, - "category": "Feasibility Study Report", - "routing_category": "generic", - "language": "zh", - "rationale": "scanned PDF not atlas", - "header_y": None, - "footer_y": None, - "next_action": "verdict_now", - "grep_query": "", - } - ) - profile, decision = _parse_profile_and_decision(raw) - assert profile.is_scanned is True - assert decision.action == "tool_call" - assert decision.tool_name == "propose.shard_plan" - assert decision.verdict is None - - -def test_planner_ready_to_shard_proposes_shard_plan() -> None: - raw = json.dumps( - { - "is_scanned": False, - "category": "Report", - "routing_category": "generic", - "language": "en", - "rationale": "enough evidence", - "next_action": "ready_to_shard", - } - ) - _profile, decision = _parse_profile_and_decision(raw) - assert decision.action == "tool_call" - assert decision.tool_name == "propose.shard_plan" - - -def test_planner_legacy_inspect_more_falls_through_to_ready_to_shard() -> None: - raw = json.dumps( - { - "is_scanned": False, - "category": "Report", - "routing_category": "generic", - "language": "en", - "rationale": "need more pages", - "next_action": "inspect_more", - "inspect_pages": [3, 8], - } - ) - _profile, decision = _parse_profile_and_decision(raw) - assert decision.tool_name == "propose.shard_plan" - assert decision.tool_args == {} - - -def test_executor_legacy_verdict_now_without_status_becomes_shard() -> None: - decision = _parse_decision( - json.dumps( - { - "action": "verdict_now", - "rationale": "classification done", - } - ) - ) - assert decision.action == "tool_call" - assert decision.tool_name == "propose.shard_plan" - - -def test_executor_legacy_verdict_now_with_abort_status_uses_verdict_tool() -> None: - decision = _parse_decision( - json.dumps( - { - "action": "verdict_now", - "rationale": "cannot profile", - "verdict": {"status": "abort", "rationale": "cannot profile"}, - } - ) - ) - assert decision.action == "tool_call" - assert decision.tool_name == "verdict" - assert decision.tool_args["status"] == "abort" - - -def _seed_pages(blackboard: AgentBlackboard, page_count: int) -> None: - from app.services.document_agent.manifest import PageFeature, PageLabel - - blackboard.page_count = page_count - blackboard.doc_stats = {"page_count": page_count} - blackboard.page_features = [ - PageFeature( - page=page, - raw_text_length=0, - text_density=0.0, - image_coverage=1.0, - image_count=1, - table_count=0, - drawings_count=0, - orientation="portrait", - width=612.0, - height=792.0, - has_asset=True, - is_blank_like=True, - ) - for page in range(1, page_count + 1) - ] - blackboard.page_labels = [ - PageLabel(page=page, kind="normal", confidence=0.9) - for page in range(1, page_count + 1) - ] - - -def test_executor_initial_ready_to_shard_reaches_success_without_abort() -> None: - blackboard = AgentBlackboard() - _seed_pages(blackboard, 4) - blackboard.document_profile = DocumentProfile( - is_scanned=True, - category="Feasibility Study Report", - routing_category="generic", - rationale="scanned PDF not atlas", - ) - from app.services.document_agent.manifest import TocResult - - blackboard.toc_result = TocResult(method="none", notes="no toc") - ctx = ToolContext( - pdf_path="/tmp/scanned.pdf", - job_id="job-scanned", - blackboard=blackboard, - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), - trace=None, - settings={}, # deterministic executor (no LLM) - ) - initial = ReflexionDecision( - action="tool_call", - rationale="ready", - tool_name="propose.shard_plan", - tool_args={}, - ) - result = ReActExecutor( - ctx, - registry=REGISTRY, - max_rounds=10, - initial_decision=initial, - ).run() - assert result.verdict.status == "success" - assert blackboard.shard_plan is not None - assert len(blackboard.shard_plan.shards) >= 1 - - -def test_executor_empty_initial_tool_falls_through_to_success() -> None: - """Missing tool_name must coerce to propose.shard_plan, not abort.""" - blackboard = AgentBlackboard() - _seed_pages(blackboard, 3) - from app.services.document_agent.manifest import TocResult - - blackboard.toc_result = TocResult(method="none", notes="no toc") - blackboard.document_profile = DocumentProfile( - is_scanned=True, - category="Report", - routing_category="generic", - ) - ctx = ToolContext( - pdf_path="/tmp/scanned.pdf", - job_id="job-legacy", - blackboard=blackboard, - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), - trace=None, - settings={}, - ) - initial = ReflexionDecision( - action="tool_call", - rationale="stale empty decision", - tool_name=None, - tool_args={}, - ) - result = ReActExecutor( - ctx, - registry=REGISTRY, - max_rounds=10, - initial_decision=initial, - ).run() - assert result.verdict.status == "success" - assert blackboard.shard_plan is not None - assert len(blackboard.shard_plan.shards) == 1 diff --git a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py index 788836bdd..fc10dcaa8 100644 --- a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py +++ b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py @@ -12,7 +12,6 @@ os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") os.environ.setdefault("S3_TEMP_PATH", "/tmp") -from app.services.document_agent.budget import BudgetTracker from app.services.document_agent.manifest import ( PageAnatomyMap, PageFeature, @@ -22,14 +21,13 @@ TocResult, ToolContext, ) -from app.services.document_agent.state import AgentBlackboard +from app.services.document_agent.state import ProfileBlackboard from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, serialize_skeleton_anchor, serialize_title_node, ) from app.services.document_agent.structure.hierarchy_locator import ( - ResolvedHierarchyRange, TitleMatch, TitleNode, ) @@ -45,8 +43,7 @@ def _ctx(*, page_count: int = 10) -> ToolContext: return ToolContext( pdf_path="/tmp/doc.pdf", job_id="job-wire", - blackboard=AgentBlackboard(page_count=page_count), - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), + blackboard=ProfileBlackboard(page_count=page_count), trace=None, settings={}, ) @@ -75,10 +72,8 @@ def _anchor(*, title: str = "Ch1", page: int = 2) -> SkeletonAnchor: match_overrides={ (title,): TitleMatch( page=page, - confidence=1.0, source="anchored", matched_line=title, - score=1.0, candidates=[page], evidence={}, ) @@ -86,7 +81,7 @@ def _anchor(*, title: str = "Ch1", page: int = 2) -> SkeletonAnchor: null_page_report=[], bulk_count=1, pruned_count=0, - locate_agent="offset_guided_bulk", + locate_method="offset_guided_bulk", ) @@ -113,7 +108,7 @@ def _anatomy(*, with_anchor: bool) -> PageAnatomyMap: for page in range(1, 11) ], "page_labels": [ - PageLabel(page=page, kind="normal", confidence=1.0) + PageLabel(page=page, kind="normal") for page in range(1, 11) ], "toc_result": TocResult(method="vlm_batch", toc_pages=[1]), @@ -128,12 +123,10 @@ def _anatomy(*, with_anchor: bool) -> PageAnatomyMap: page_offset=0, anchor_type="forced_max_size", anchor_evidence="test", - confidence=1.0, ) ], ), "toc_hierarchies": _toc(), - "toc_page_offset": 0 if with_anchor else None, } if with_anchor: kwargs["skeleton_anchor"] = serialize_skeleton_anchor(_anchor()) @@ -151,12 +144,11 @@ def fake_anchor_hierarchy(**_kwargs): return [_node()], _anchor() with patch( - "app.services.document_agent.agents.calibration.orchestrator.anchor_hierarchy", + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", side_effect=fake_anchor_hierarchy, ): run_toc_anchoring(ctx) - assert ctx.blackboard.toc_page_offset == 0 assert isinstance(ctx.blackboard.skeleton_anchor, dict) assert ctx.blackboard.skeleton_anchor["offset"] == 0 assert ctx.blackboard.skeleton_nodes @@ -172,15 +164,15 @@ def _boom(*_args, **_kwargs): with ( patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", side_effect=_boom, ), patch( - "app.services.document_agent.agents.calibration.orchestrator.anchor_hierarchy", + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", side_effect=_boom, ), patch( - "app.services.document_agent.agents.calibration.procedure.finalize_calibration_result", + "app.services.document_agent.calibration.procedure.finalize_calibration_result", side_effect=_boom, ), ): @@ -207,7 +199,6 @@ def test_shard_plan_reads_offset_and_does_not_calibrate() -> None: ], } ] - ctx.blackboard.toc_page_offset = 0 ctx.blackboard.skeleton_nodes = [ serialize_title_node(TitleNode(title="Ch1", level=1, printed_page=3)), serialize_title_node(TitleNode(title="Ch2", level=1, printed_page=120)), @@ -219,19 +210,15 @@ def test_shard_plan_reads_offset_and_does_not_calibrate() -> None: match_overrides={ ("Ch1",): TitleMatch( page=3, - confidence=1.0, source="anchored", matched_line="Ch1", - score=1.0, candidates=[3], evidence={}, ), ("Ch2",): TitleMatch( page=120, - confidence=1.0, source="anchored", matched_line="Ch2", - score=1.0, candidates=[120], evidence={}, ), @@ -239,7 +226,7 @@ def test_shard_plan_reads_offset_and_does_not_calibrate() -> None: null_page_report=[], bulk_count=2, pruned_count=0, - locate_agent="offset_guided_bulk", + locate_method="offset_guided_bulk", ) ) ctx.blackboard.toc_result = TocResult(method="vlm_batch") @@ -251,13 +238,14 @@ def _boom(*_args, **_kwargs): raise AssertionError("shard plan must not recalibrate") with patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", side_effect=_boom, ): result = propose_shard_plan(ctx, {}) assert result.status == "ok" - assert ctx.blackboard.toc_page_offset == 0 + assert ctx.blackboard.skeleton_anchor is not None + assert ctx.blackboard.skeleton_anchor["offset"] == 0 assert ctx.blackboard.shard_plan is not None @@ -280,7 +268,7 @@ def _pending_tocs() -> list[dict[str, object]]: ] -def test_profile_classifies_pending_toc_before_finalize() -> None: +def test_profile_classifies_pending_toc_after_finalize() -> None: ctx = _ctx(page_count=30) ctx.blackboard.toc_hierarchies = _pending_tocs() ctx.blackboard.toc_result = TocResult(method="vlm_batch", toc_pages=[1, 20, 21]) @@ -294,34 +282,19 @@ def fake_finalize(**kwargs): with ( patch( - "app.services.document_agent.agents.calibration.orchestrator.anchor_hierarchy", + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", return_value=([_node()], _anchor()), ), - patch.dict( - run_toc_anchoring.__globals__, - { - "resolve_hierarchy_page_ranges": lambda *_args, **_kwargs: [ - ResolvedHierarchyRange( - title="Ch1", - level=1, - start_page=2, - end_page=19, - path_titles=("Ch1",), - match=None, - ) - ] - }, - ), patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", return_value=object(), ), patch( - "app.services.document_agent.agents.calibration.procedure.pick_primary_offset", + "app.services.document_agent.calibration.procedure.pick_primary_offset", return_value=0, ), patch( - "app.services.document_agent.agents.calibration.procedure.finalize_calibration_result", + "app.services.document_agent.calibration.procedure.finalize_calibration_result", side_effect=fake_finalize, ), ): @@ -347,34 +320,19 @@ def _boom(*_args, **_kwargs): with ( patch( - "app.services.document_agent.agents.calibration.orchestrator.anchor_hierarchy", + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", return_value=([_node()], _anchor()), ), - patch.dict( - run_toc_anchoring.__globals__, - { - "resolve_hierarchy_page_ranges": lambda *_args, **_kwargs: [ - ResolvedHierarchyRange( - title="Ch1", - level=1, - start_page=2, - end_page=19, - path_titles=("Ch1",), - match=None, - ) - ] - }, - ), patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", return_value=object(), ), patch( - "app.services.document_agent.agents.calibration.procedure.pick_primary_offset", + "app.services.document_agent.calibration.procedure.pick_primary_offset", return_value=0, ), patch( - "app.services.document_agent.agents.calibration.procedure.finalize_calibration_result", + "app.services.document_agent.calibration.procedure.finalize_calibration_result", side_effect=_boom, ), ): @@ -409,7 +367,7 @@ def test_c4_uses_persisted_pending_relationship_and_does_not_classify() -> None: for page in range(1, 31) ] anatomy.page_labels = [ - PageLabel(page=page, kind="normal", confidence=1.0) + PageLabel(page=page, kind="normal") for page in range(1, 31) ] anatomy.toc_hierarchies = _pending_tocs() diff --git a/apps/worker/tests/contract/test_propose_shard_plan_contract.py b/apps/worker/tests/contract/test_propose_shard_plan_contract.py index 5953f341a..2b870c875 100644 --- a/apps/worker/tests/contract/test_propose_shard_plan_contract.py +++ b/apps/worker/tests/contract/test_propose_shard_plan_contract.py @@ -9,9 +9,8 @@ os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") os.environ.setdefault("S3_TEMP_PATH", "/tmp") -from app.services.document_agent.budget import BudgetTracker from app.services.document_agent.manifest import PageFeature, TocResult, ToolContext -from app.services.document_agent.state import AgentBlackboard +from app.services.document_agent.state import ProfileBlackboard from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, serialize_skeleton_anchor, @@ -43,8 +42,7 @@ def _ctx(*, page_count: int, blank_pages: list[int] | None = None) -> ToolContex ctx = ToolContext( pdf_path="/tmp/doc.pdf", job_id="job-shard", - blackboard=AgentBlackboard(page_count=page_count), - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), + blackboard=ProfileBlackboard(page_count=page_count), trace=None, settings={ "shard_threshold": 200, @@ -62,10 +60,8 @@ def _ctx(*, page_count: int, blank_pages: list[int] | None = None) -> ToolContex def _match(title: str, page: int) -> TitleMatch: return TitleMatch( page=page, - confidence=1.0, source="anchored", matched_line=title, - score=1.0, candidates=[page], evidence={}, ) @@ -91,7 +87,7 @@ def _seed_skeleton( null_page_report=[], bulk_count=len(overrides), pruned_count=0, - locate_agent="offset_guided_bulk", + locate_method="offset_guided_bulk", ) ) if pending_records is not None: @@ -147,6 +143,63 @@ def test_hierarchy_pack_cuts_before_next_chapter() -> None: assert plan.validation.valid is True +def test_shard_plan_attaches_calibrated_toc_hierarchies_per_shard() -> None: + ctx = _ctx(page_count=250) + _seed_skeleton( + ctx, + nodes=[ + TitleNode( + title="Ch1", + level=1, + printed_page=3, + children=[ + TitleNode(title="1.1", level=2, printed_page=3), + TitleNode(title="1.2", level=2, printed_page=80), + ], + ), + TitleNode(title="Ch2", level=1, printed_page=120), + ], + overrides={ + ("Ch1",): 3, + ("Ch1", "1.1"): 3, + ("Ch1", "1.2"): 80, + ("Ch2",): 120, + }, + hierarchies=[ + { + "toc_range": [1, 2], + "toc_range_unit": "page", + "toc_with_level": [ + {"heading": "Ch1", "level": 1, "page_number": 3}, + {"heading": "1.1", "level": 2, "page_number": 3}, + {"heading": "1.2", "level": 2, "page_number": 80}, + {"heading": "Ch2", "level": 1, "page_number": 120}, + ], + } + ], + ) + + propose_shard_plan(ctx, {}) + plan = ctx.blackboard.shard_plan + assert plan is not None + assert len(plan.shards) == 2 + + first = plan.shards[0].toc_hierarchies + second = plan.shards[1].toc_hierarchies + assert first is not None + assert second is not None + assert first[0]["toc_range"] == [1, 119] + assert second[0]["toc_range"] == [120, 250] + assert [row["heading"] for row in first[0]["toc_with_level"]] == [ + "Ch1", + "1.1", + "1.2", + ] + assert [row["heading"] for row in second[0]["toc_with_level"]] == ["Ch2"] + assert all("page_number" not in row for row in first[0]["toc_with_level"]) + assert all("page_number" not in row for row in second[0]["toc_with_level"]) + + def test_hierarchy_pack_keeps_same_parent_siblings_together() -> None: ctx = _ctx(page_count=250) _seed_skeleton( @@ -302,7 +355,7 @@ def test_pending_toc_forest_is_packed_separately() -> None: null_page_report=[], bulk_count=1, pruned_count=0, - locate_agent="offset_guided_bulk", + locate_method="offset_guided_bulk", ) ), } @@ -365,7 +418,7 @@ def test_contained_pending_toc_does_not_cut() -> None: null_page_report=[], bulk_count=1, pruned_count=0, - locate_agent="offset_guided_bulk", + locate_method="offset_guided_bulk", ) ), } @@ -381,6 +434,106 @@ def test_contained_pending_toc_does_not_cut() -> None: ] +def _deep_tree_nodes() -> list[TitleNode]: + return [ + TitleNode( + title="Ch1", + level=1, + printed_page=1, + children=[ + TitleNode(title="1.1", level=2, printed_page=2), + TitleNode(title="1.6", level=2, printed_page=30), + ], + ), + TitleNode( + title="Ch2", + level=1, + printed_page=31, + children=[ + TitleNode( + title="S1", + level=2, + printed_page=31, + children=[TitleNode(title="S1.1", level=3, printed_page=31)], + ), + TitleNode( + title="S9", + level=2, + printed_page=210, + children=[TitleNode(title="S9.1", level=3, printed_page=210)], + ), + ], + ), + ] + + +_DEEP_TREE_HIERARCHIES: list[dict[str, object]] = [ + { + "toc_range": [1, 1], + "toc_range_unit": "page", + "toc_with_level": [ + {"heading": "Ch1", "level": 1, "page_number": 1}, + {"heading": "1.1", "level": 2, "page_number": 2}, + {"heading": "1.6", "level": 2, "page_number": 30}, + {"heading": "Ch2", "level": 1, "page_number": 31}, + {"heading": "S1", "level": 2, "page_number": 31}, + {"heading": "S1.1", "level": 3, "page_number": 31}, + {"heading": "S9", "level": 2, "page_number": 210}, + {"heading": "S9.1", "level": 3, "page_number": 210}, + ], + } +] + +_DEEP_TREE_LEAF_OVERRIDES: dict[tuple[str, ...], int] = { + ("Ch1", "1.1"): 2, + ("Ch1", "1.6"): 30, + ("Ch2", "S1", "S1.1"): 31, + ("Ch2", "S9", "S9.1"): 210, +} + + +def test_shard_toc_omits_unrelated_node_when_parents_unanchored() -> None: + ctx = _ctx(page_count=250) + _seed_skeleton( + ctx, + nodes=_deep_tree_nodes(), + overrides=dict(_DEEP_TREE_LEAF_OVERRIDES), + hierarchies=_DEEP_TREE_HIERARCHIES, + ) + + propose_shard_plan(ctx, {}) + plan = ctx.blackboard.shard_plan + assert plan is not None + + tail = plan.shards[-1].toc_hierarchies + assert tail is not None + headings = [row["heading"] for row in tail[0]["toc_with_level"]] + assert headings == ["S9.1"] + + +def test_shard_toc_reopens_anchored_ancestors() -> None: + ctx = _ctx(page_count=250) + _seed_skeleton( + ctx, + nodes=_deep_tree_nodes(), + overrides={ + **_DEEP_TREE_LEAF_OVERRIDES, + ("Ch2",): 31, + ("Ch2", "S9"): 210, + }, + hierarchies=_DEEP_TREE_HIERARCHIES, + ) + + propose_shard_plan(ctx, {}) + plan = ctx.blackboard.shard_plan + assert plan is not None + + tail = plan.shards[-1].toc_hierarchies + assert tail is not None + headings = [row["heading"] for row in tail[0]["toc_with_level"]] + assert headings == ["Ch2", "S9", "S9.1"] + + def test_fat_leaf_uses_blank_page_in_window() -> None: ctx = _ctx(page_count=450, blank_pages=[195]) _seed_skeleton( diff --git a/apps/worker/tests/contract/test_select_global_toc_hierarchies_contract.py b/apps/worker/tests/contract/test_select_global_toc_hierarchies_contract.py new file mode 100644 index 000000000..9f461591e --- /dev/null +++ b/apps/worker/tests/contract/test_select_global_toc_hierarchies_contract.py @@ -0,0 +1,46 @@ +"""Contracts for primary vs pending TOC split (no proximity merge).""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from app.services.document_agent.structure.toc_anchoring import ( + select_global_toc_hierarchies, +) + + +def test_nearby_second_toc_is_pending_not_merged_into_primary() -> None: + """Former front-cluster gap (≤5 pages) must not pull a later TOC into primary.""" + hierarchies = [ + { + "toc_range": [1, 1], + "toc_range_unit": "page", + "toc_with_level": [{"heading": "Front", "level": 1, "page_number": 2}], + }, + { + "toc_range": [4, 4], + "toc_range_unit": "page", + "toc_with_level": [{"heading": "Near", "level": 1, "page_number": 5}], + }, + { + "toc_range": [40, 40], + "toc_range_unit": "page", + "toc_with_level": [{"heading": "Ch3", "level": 1, "page_number": 1}], + }, + ] + primary, pending, summary = select_global_toc_hierarchies( + hierarchies=hierarchies, + filename="doc.pdf", + ) + assert primary == [hierarchies[0]] + assert pending == [hierarchies[1], hierarchies[2]] + assert summary["strategy"] == "earliest_forest_rest_pending" + assert summary["primary_count"] == 1 + assert summary["pending_count"] == 2 diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index 11008b5cd..e87104b4a 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -1,12 +1,14 @@ -"""Contract tests for structure_anchoring (Phase-2) + calibrate wiring.""" +"""Contract tests for hierarchy anchoring (Phase-2) + calibrate wiring.""" from __future__ import annotations import os from collections.abc import Callable, Iterator from contextlib import contextmanager -from typing import Any from unittest.mock import patch +from typing import Any + +import pytest os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") @@ -15,44 +17,42 @@ os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") os.environ.setdefault("S3_TEMP_PATH", "/tmp") -from app.services.document_agent.budget import BudgetTracker from app.services.document_agent.manifest import ToolContext -from app.services.document_agent.state import AgentBlackboard +from app.services.document_agent.state import ProfileBlackboard from app.services.document_agent.structure.hierarchy_locator import TitleMatch, TitleNode -from app.services.document_agent.structure import structure_anchoring as anchoring +from app.services.document_agent.structure import anchoring_primitives as anchoring + + +@pytest.fixture(autouse=True) +def _rebind_live_anchoring_primitives() -> Iterator[None]: + """Rebind after contract fixtures that clear ``app.*`` from ``sys.modules``.""" + global anchoring + from app.services.document_agent.structure import anchoring_primitives as live + + anchoring = live + yield @contextmanager def _patch_verify(fake_verify: Callable[..., dict[str, Any]]) -> Iterator[None]: - """Patch verify on the module dict closed over by live anchoring code.""" - from app.services.document_agent.agents.calibration import procedure - - dicts = [procedure.offset_guided_anchoring.__globals__, anchoring.__dict__] - seen: set[int] = set() - originals: list[tuple[dict[str, Any], Any]] = [] - for module_dict in dicts: - dict_id = id(module_dict) - if dict_id in seen: - continue - seen.add(dict_id) - originals.append((module_dict, module_dict.get("verify_section_page_choice"))) - module_dict["verify_section_page_choice"] = fake_verify - try: + """Patch verify on the live anchoring module (not a collection-time zombie). + + Contract fixtures that call ``clear_application_modules()`` drop ``app.*`` from + ``sys.modules``. Resolve the patch target by dotted path at enter time so it + always hits the live globals closed over by ``_vlm_confirm_single_page``. + """ + with patch( + "app.services.document_agent.structure.anchoring_primitives.verify_section_page_choice", + fake_verify, + ): yield - finally: - for module_dict, original in originals: - if original is None: - module_dict.pop("verify_section_page_choice", None) - else: - module_dict["verify_section_page_choice"] = original def _ctx() -> ToolContext: return ToolContext( pdf_path="/tmp/doc.pdf", job_id="job-anchor", - blackboard=AgentBlackboard(), - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), + blackboard=ProfileBlackboard(), trace=None, settings={"vlm_model": "test-vlm"}, ) @@ -121,6 +121,134 @@ def test_null_page_parent_located_via_compact_text() -> None: assert overrides[("1 Overview",)].page == 5 assert report[0]["result"] != "unresolved" assert report[0]["page"] == 5 + assert report[0]["window"] == [1, 5] + + +def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: + """No left sibling: miss text → ``scan_title_forward`` within 2+4+6+10 budget.""" + child = TitleNode(title="22.1 Intro", level=2, printed_page=278, children=[]) + parent = TitleNode( + title="Chapter 22", + level=1, + printed_page=None, + children=[child], + ) + leaf_match = { + ("Chapter 22", "22.1 Intro"): TitleMatch( + page=278, + source="test", + matched_line="", + candidates=[278], + evidence={}, + ) + } + body_pages = list(range(1, 301)) + page_texts = {page: "noise" for page in body_pages} + ctx = _ctx() + + scanned_starts: list[int] = [] + + def fake_scan(**kwargs: Any) -> Any: + from app.services.document_agent.calibration.scan import TitleScanResult + + scanned_starts.append(int(kwargs["start_page"])) + assert int(kwargs["page_count"]) == 278 + assert int(kwargs["start_page"]) == anchoring._first_sibling_null_parent_scan_start( + 278 + ) + return TitleScanResult( + title=str(kwargs["title"]), + found=True, + found_page=270, + scanned_pages=list(range(int(kwargs["start_page"]), 271)), + next_start=271, + ) + + with patch( + "app.services.document_agent.calibration.scan.scan_title_forward", + side_effect=fake_scan, + ): + with patch.object(anchoring, "_visual_rtl_locate_parent") as rtl: + overrides, report = anchoring.locate_null_page_parent_overrides( + nodes=[parent], + match_overrides=leaf_match, + page_texts=page_texts, + body_pages=body_pages, + ctx=ctx, + ) + rtl.assert_not_called() + + assert scanned_starts == [anchoring._first_sibling_null_parent_scan_start(278)] + assert overrides[("Chapter 22",)].page == 270 + assert report[0]["accept"] == "scan_forward" + assert report[0]["window"] == [ + anchoring._first_sibling_null_parent_scan_start(278), + 278, + ] + + +def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: + left_child = TitleNode(title="A.1", level=2, printed_page=10, children=[]) + left = TitleNode(title="A", level=1, printed_page=10, children=[left_child]) + right_child = TitleNode(title="B.1", level=2, printed_page=50, children=[]) + right = TitleNode(title="B", level=1, printed_page=None, children=[right_child]) + overrides_in = { + ("A",): TitleMatch( + page=10, + source="test", + matched_line="", + candidates=[10], + evidence={}, + ), + ("A", "A.1"): TitleMatch( + page=10, + source="test", + matched_line="", + candidates=[10], + evidence={}, + ), + ("B", "B.1"): TitleMatch( + page=50, + source="test", + matched_line="", + candidates=[50], + evidence={}, + ), + } + page_texts = {p: "noise" for p in range(1, 61)} + ctx = _ctx() + + def fake_rtl(**kwargs: Any) -> tuple[TitleMatch, int]: + assert kwargs["left"] == 10 + assert kwargs["right"] == 50 + return ( + TitleMatch( + page=40, + source="inspect_vlm", + matched_line="", + candidates=[40], + evidence={"accept": "visual_rtl"}, + ), + 3, + ) + + with patch( + "app.services.document_agent.calibration.scan.scan_title_forward" + ) as scan: + with patch.object( + anchoring, "_visual_rtl_locate_parent", side_effect=fake_rtl + ): + overrides, report = anchoring.locate_null_page_parent_overrides( + nodes=[left, right], + match_overrides=overrides_in, + page_texts=page_texts, + body_pages=list(range(1, 61)), + ctx=ctx, + ) + scan.assert_not_called() + + assert overrides[("B",)].page == 40 + assert report[0]["accept"] == "visual_rtl" def test_phase2_bulk_via_mocked_offset() -> None: @@ -133,10 +261,8 @@ def test_phase2_bulk_via_mocked_offset() -> None: seed = { ("Intro",): TitleMatch( page=5, - confidence=0.9, - source="agent_vlm", + source="inspect_vlm", matched_line="", - score=0.9, candidates=[5], evidence={"calibration": True, "printed_page": 3}, ) @@ -144,7 +270,7 @@ def test_phase2_bulk_via_mocked_offset() -> None: def fake_verify(**kwargs: Any) -> dict[str, Any]: expected = kwargs["candidate_matches"][0].page - return {"selected_page": expected, "confidence": 0.9, "reason": "ok"} + return {"selected_page": expected, "reason": "ok"} with _patch_verify(fake_verify): matches = anchoring.offset_guided_anchoring( @@ -173,7 +299,12 @@ def test_anchor_hierarchy_uses_calibration_phase1() -> None: ] ctx = _ctx() - from app.services.document_agent.agents.calibration.types import ( + # Contract conftest evicts cached ``app.*`` modules, so resolve the live + # orchestrator inside the test rather than at import time. + from app.services.document_agent.calibration.orchestrator import ( + anchor_hierarchy, + ) + from app.services.document_agent.calibration.types import ( CalibrationRegime, CalibrationResult, CalibrationSample, @@ -198,16 +329,16 @@ def test_anchor_hierarchy_uses_calibration_phase1() -> None: def fake_verify(**kwargs: Any) -> dict[str, Any]: expected = kwargs["candidate_matches"][0].page - return {"selected_page": expected, "confidence": 0.95, "reason": "ok"} + return {"selected_page": expected, "reason": "ok"} with ( patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", return_value=phase1, ), _patch_verify(fake_verify), ): - nodes, anchor = anchoring.anchor_hierarchy( + nodes, anchor = anchor_hierarchy( nodes=leaves, toc_hierarchies=toc_hierarchies, page_texts={4: "Only\ntext"}, @@ -247,12 +378,198 @@ def test_prune_unanchored_suffix_removes_toc_leaves() -> None: assert ("A",) in overrides and ("B",) in overrides +def test_bisect_all_fail_returns_minus_one() -> None: + """No confirmed leaf under the offset ⇒ breakpoint is -1, not index 0.""" + leaves = [ + (("C1",), _leaf("C1", 1)), + (("C2",), _leaf("C2", 2)), + (("C3",), _leaf("C3", 3)), + ] + + def fake_verify(**kwargs: Any) -> dict[str, Any]: + return {"selected_page": None, "reason": "miss"} + + with _patch_verify(fake_verify): + bp = anchoring._bisect_offset_breakpoint( + leaves=leaves, + offset=0, + ctx=_ctx(), + page_count=10, + ) + assert bp == -1 + + +def test_phase2_all_bisect_fail_does_not_invent_first_leaf() -> None: + """When every Phase-2 probe fails, do not bulk-anchor the first TOC leaf.""" + from app.services.document_agent.calibration.procedure import ( + anchor_hierarchy_from_regimes, + ) + from app.services.document_agent.calibration.scan import TitleScanResult + from app.services.document_agent.calibration.types import ( + CalibrationRegime, + CalibrationResult, + CalibrationSample, + ) + + leaves = [ + _leaf("Ch1", 1), + _leaf("Ch2", 5), + _leaf("Ch3", 20), + ] + # Phase-1 sample is a different title so Ch1 is not protected by seed. + phase1 = CalibrationResult( + status="ok", + offset=10, + regimes=[ + CalibrationRegime( + kind="decimal", + offset=10, + offset_status="ok", + entry_indices=[0, 1, 2], + samples=[CalibrationSample(title="Other", physical=99)], + ) + ], + ) + ctx = _ctx() + + def fake_verify(**kwargs: Any) -> dict[str, Any]: + return {"selected_page": None, "reason": "miss"} + + def fake_scan(**kwargs: Any) -> TitleScanResult: + return TitleScanResult( + title=str(kwargs.get("title") or ""), + found=False, + found_page=None, + scanned_pages=[], + next_start=None, + ) + + with ( + _patch_verify(fake_verify), + patch( + "app.services.document_agent.calibration.scan.scan_title_forward", + fake_scan, + ), + ): + working, anchor = anchor_hierarchy_from_regimes( + nodes=leaves, + result=phase1, + entries=[ + {"heading": "Ch1", "level": 1, "page_number": 1}, + {"heading": "Ch2", "level": 1, "page_number": 5}, + {"heading": "Ch3", "level": 1, "page_number": 20}, + ], + page_texts={11: "noise", 15: "noise", 30: "noise"}, + body_pages=list(range(1, 50)), + page_count=50, + ctx=ctx, + ) + + assert working == [] + assert ("Ch1",) not in anchor.match_overrides + assert ("Ch2",) not in anchor.match_overrides + assert ("Ch3",) not in anchor.match_overrides + assert anchor.pruned_count >= 3 + + +def test_phase2_recalibrate_uses_forward_scan_beyond_plus_five() -> None: + """Breakpoint suffix reuses Phase-1 forward scan (not a +1..+5 grid).""" + from app.services.document_agent.calibration.procedure import ( + anchor_hierarchy_from_regimes, + ) + from app.services.document_agent.calibration.scan import TitleScanResult + from app.services.document_agent.calibration.types import ( + CalibrationRegime, + CalibrationResult, + CalibrationSample, + ) + + leaves = [ + _leaf("Ch1", 1), + _leaf("Ch2", 5), + _leaf("Ch3", 20), + _leaf("Ch4", 30), + ] + phase1 = CalibrationResult( + status="ok", + offset=10, + regimes=[ + CalibrationRegime( + kind="decimal", + offset=10, + offset_status="ok", + entry_indices=[0, 1, 2, 3], + samples=[CalibrationSample(title="Ch1", physical=11)], + ) + ], + ) + ctx = _ctx() + + def fake_verify(**kwargs: Any) -> dict[str, Any]: + expected = int(kwargs["candidate_matches"][0].page) + title = str(kwargs.get("title") or "") + # Prefix at offset=10; after forward-scan recalibrate, suffix uses +16. + ok = { + ("Ch1", 11), + ("Ch2", 15), + ("Ch3", 36), # 20+16 + ("Ch4", 46), # 30+16 + } + if (title, expected) in ok: + return {"selected_page": expected, "reason": "ok"} + return {"selected_page": None, "reason": "miss"} + + def fake_scan(**kwargs: Any) -> TitleScanResult: + title = str(kwargs.get("title") or "") + start = int(kwargs.get("start_page") or 0) + # Old slot for Ch3 was page 30; scan starts at 31 and finds 36 → offset 16. + assert title == "Ch3" + assert start == 31 + return TitleScanResult( + title=title, + found=True, + found_page=36, + scanned_pages=[31, 32, 33, 34, 35, 36], + next_start=37, + ) + + with ( + _patch_verify(fake_verify), + patch( + "app.services.document_agent.calibration.scan.scan_title_forward", + fake_scan, + ), + ): + working, anchor = anchor_hierarchy_from_regimes( + nodes=leaves, + result=phase1, + entries=[ + {"heading": "Ch1", "level": 1, "page_number": 1}, + {"heading": "Ch2", "level": 1, "page_number": 5}, + {"heading": "Ch3", "level": 1, "page_number": 20}, + {"heading": "Ch4", "level": 1, "page_number": 30}, + ], + page_texts={11: "Ch1", 15: "Ch2", 36: "Ch3", 46: "Ch4"}, + body_pages=list(range(1, 60)), + page_count=60, + ctx=ctx, + ) + + titles = [n.title for n in working] + assert titles == ["Ch1", "Ch2", "Ch3", "Ch4"] + assert anchor.match_overrides[("Ch1",)].page == 11 + assert anchor.match_overrides[("Ch2",)].page == 15 + assert anchor.match_overrides[("Ch3",)].page == 36 # 20+16 + assert anchor.match_overrides[("Ch4",)].page == 46 # 30+16 + + def test_phase2_recalibrate_miss_drops_suffix_from_tree() -> None: """When suffix cannot be recalibrated, those leaves leave the TOC tree.""" - from app.services.document_agent.agents.calibration.procedure import ( + from app.services.document_agent.calibration.procedure import ( anchor_hierarchy_from_regimes, ) - from app.services.document_agent.agents.calibration.types import ( + from app.services.document_agent.calibration.scan import TitleScanResult + from app.services.document_agent.calibration.types import ( CalibrationRegime, CalibrationResult, CalibrationSample, @@ -284,14 +601,28 @@ def test_phase2_recalibrate_miss_drops_suffix_from_tree() -> None: def fake_verify(**kwargs: Any) -> dict[str, Any]: expected = int(kwargs["candidate_matches"][0].page) title = str(kwargs.get("title") or "") - # Prefix Ch1/Ch2 at offset=10 confirm; Ch3/Ch4 and recalibrate (+1..+5) miss. + # Prefix Ch1/Ch2 at offset=10 confirm; Ch3/Ch4 miss under old offset. ok_pages = {11, 15} # 1+10, 5+10 if expected in ok_pages and title in {"Ch1", "Ch2"}: - return {"selected_page": expected, "confidence": 0.95, "reason": "ok"} - # Tail / bisect mid / recalibrate probes for Ch3/Ch4 all fail. - return {"selected_page": None, "confidence": 0.1, "reason": "miss"} + return {"selected_page": expected, "reason": "ok"} + return {"selected_page": None, "reason": "miss"} + + def fake_scan(**kwargs: Any) -> TitleScanResult: + return TitleScanResult( + title=str(kwargs.get("title") or ""), + found=False, + found_page=None, + scanned_pages=[], + next_start=None, + ) - with _patch_verify(fake_verify): + with ( + _patch_verify(fake_verify), + patch( + "app.services.document_agent.calibration.scan.scan_title_forward", + fake_scan, + ), + ): working, anchor = anchor_hierarchy_from_regimes( nodes=leaves, result=phase1, @@ -316,12 +647,57 @@ def fake_verify(**kwargs: Any) -> dict[str, Any]: assert anchor.pruned_count >= 2 +def _printed_page_parent(title: str, printed: int, child: TitleNode) -> TitleNode: + return TitleNode(title=title, level=1, printed_page=printed, children=[child]) + + +def test_parent_backfill_uses_descendant_regime_offset() -> None: + from app.services.document_agent.structure import anchoring_primitives as primitives + + early_child = TitleNode(title="A.1", level=2, printed_page=12, children=[]) + late_child = TitleNode(title="B.1", level=2, printed_page=42, children=[]) + section_a = _printed_page_parent("Section A", 10, early_child) + section_b = _printed_page_parent("Section B", 40, late_child) + matches = { + **primitives.bulk_offset_matches([(("Section A", "A.1"), early_child)], 5), + **primitives.bulk_offset_matches([(("Section B", "B.1"), late_child)], 9), + } + + parents = primitives.backfill_parent_offset_matches( + nodes=[section_a, section_b], + matches=matches, + page_count=60, + ) + + assert parents[("Section A",)].page == 15 + assert parents[("Section B",)].page == 49 + assert parents[("Section A",)].evidence["parent_backfill"] is True + + +def test_parent_backfill_skips_unanchored_and_out_of_range() -> None: + from app.services.document_agent.structure import anchoring_primitives as primitives + + tail_child = TitleNode(title="Tail.1", level=2, printed_page=96, children=[]) + ghost_child = TitleNode(title="Ghost.1", level=2, printed_page=11, children=[]) + tail = _printed_page_parent("Tail", 95, tail_child) + ghost = _printed_page_parent("Ghost", 10, ghost_child) + matches = primitives.bulk_offset_matches([(("Tail", "Tail.1"), tail_child)], 8) + + parents = primitives.backfill_parent_offset_matches( + nodes=[tail, ghost], + matches=matches, + page_count=100, + ) + + assert parents == {} + + def test_multi_regime_phase2_merges_physical_overrides() -> None: """Roman + decimal + prefixed each apply their own offset → physical pages.""" - from app.services.document_agent.agents.calibration.procedure import ( + from app.services.document_agent.calibration.procedure import ( anchor_hierarchy_from_regimes, ) - from app.services.document_agent.agents.calibration.types import ( + from app.services.document_agent.calibration.types import ( CalibrationRegime, CalibrationResult, CalibrationSample, @@ -383,7 +759,7 @@ def test_multi_regime_phase2_merges_physical_overrides() -> None: def fake_verify(**kwargs: Any) -> dict[str, Any]: expected = kwargs["candidate_matches"][0].page - return {"selected_page": expected, "confidence": 0.95, "reason": "ok"} + return {"selected_page": expected, "reason": "ok"} with _patch_verify(fake_verify): _working, anchor = anchor_hierarchy_from_regimes( diff --git a/apps/worker/tests/contract/test_toc_confirm_batch_contract.py b/apps/worker/tests/contract/test_toc_confirm_batch_contract.py index f92992519..6de067af1 100644 --- a/apps/worker/tests/contract/test_toc_confirm_batch_contract.py +++ b/apps/worker/tests/contract/test_toc_confirm_batch_contract.py @@ -8,7 +8,6 @@ import pytest -from app.services.document_agent.budget import BudgetTracker, StageEnvelope from app.services.document_agent.manifest import TocAnchorPage from app.services.document_agent.tools import extract_toc_with_boundaries as toc_tool @@ -79,17 +78,9 @@ def chat_completion_with_usage(self, **kwargs: Any) -> tuple[str, dict[str, int] lambda requested_model=None: (_FakeClient(), requested_model or "fake-vlm"), ) - budget = BudgetTracker( - plan_budget=50_000, - visual_budget=200_000, - visual_stage_envelopes={ - "toc_confirm": StageEnvelope(min_guarantee=0, cap=None), - }, - ) confirmed, confirm_failed, evidence = toc_tool._vlm_confirm_anchors( # noqa: SLF001 anchors, model="fake-vlm", - budget=budget, ) assert sorted(call_pages[0] + call_pages[1]) == [5, 11, 40, 44, 55, 78, 97] @@ -120,7 +111,6 @@ def chat_completion_with_usage(self, **kwargs: Any) -> tuple[str, dict[str, int] confirmed, confirm_failed, _evidence = toc_tool._vlm_confirm_anchors( # noqa: SLF001 anchors, model="fake-vlm", - budget=None, ) assert confirmed == [] assert confirm_failed is True diff --git a/apps/worker/tests/contract/test_toc_graft_contract.py b/apps/worker/tests/contract/test_toc_graft_contract.py index 055a98e85..4777c5b9a 100644 --- a/apps/worker/tests/contract/test_toc_graft_contract.py +++ b/apps/worker/tests/contract/test_toc_graft_contract.py @@ -12,7 +12,6 @@ os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") os.environ.setdefault("S3_TEMP_PATH", "/tmp") -from app.services.document_agent.budget import BudgetTracker from app.services.document_agent.manifest import ( PageAnatomyMap, PageFeature, @@ -22,7 +21,7 @@ TocResult, ToolContext, ) -from app.services.document_agent.state import AgentBlackboard +from app.services.document_agent.state import ProfileBlackboard from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, serialize_skeleton_anchor, @@ -32,7 +31,10 @@ TitleMatch, TitleNode, ) -from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring +from app.services.document_agent.structure.toc_anchoring import ( + classify_toc_relationship, + run_toc_anchoring, +) from app.services.document_agent.structure.toc_graft import graft_contained_toc from app.services.page_memory.skeleton_extractor import extract_section_skeletons @@ -40,10 +42,8 @@ def _match(title: str, page: int) -> TitleMatch: return TitleMatch( page=page, - confidence=1.0, source="anchored", matched_line=title, - score=1.0, candidates=[page], evidence={}, ) @@ -62,14 +62,12 @@ def _graft( page_count: int = 50, ) -> object: body_pages = list(range(1, page_count + 1)) - page_texts = {page: "" for page in body_pages} return graft_contained_toc( primary_nodes=primary, primary_overrides=_overrides(primary_pages), contained_nodes=contained, contained_overrides=_overrides(contained_pages), page_count=page_count, - page_texts=page_texts, body_pages=body_pages, ) @@ -85,6 +83,37 @@ def _node_at(nodes: list[TitleNode], path: tuple[str, ...]) -> TitleNode | None: return found +def test_skip_outside_parent_coverage_records_event_without_orphan_override() -> None: + result = _graft( + primary=[ + TitleNode( + title="第一章", + level=1, + printed_page=10, + children=[TitleNode(title="1.1", level=2, printed_page=12)], + ), + TitleNode(title="第二章", level=1, printed_page=30), + ], + primary_pages={("第一章",): 10, ("第一章", "1.1"): 12, ("第二章",): 30}, + contained=[ + TitleNode( + title="第一章", + level=1, + printed_page=10, + children=[TitleNode(title="stray", level=2, printed_page=35)], + ) + ], + contained_pages={("第一章",): 10, ("第一章", "stray"): 35}, + ) + + assert [child.title for child in result.nodes[0].children] == ["1.1"] + assert ("第一章", "stray") not in result.match_overrides + skip = next(event for event in result.events if event["action"] == "skip") + assert skip["reason"] == "outside_parent_coverage" + assert skip["contained_path"] == ("第一章", "stray") + assert skip["parent_path"] == ("第一章",) + + def test_dedup_keeps_primary_title_and_hangs_child() -> None: result = _graft( primary=[TitleNode(title="第一章", level=1, printed_page=10)], @@ -126,7 +155,6 @@ def test_dedup_ignores_title_and_keeps_primary_override() -> None: ], contained_overrides=_overrides({("Chapter 1",): 10, ("Chapter 1", "1.2"): 12}), page_count=50, - page_texts={page: "" for page in body_pages}, body_pages=body_pages, ) @@ -204,7 +232,6 @@ def test_two_contained_tocs_graft_in_order() -> None: ], contained_overrides=_overrides({("第一章",): 10, ("第一章", "1.3"): 20}), page_count=50, - page_texts={page: "" for page in range(1, 51)}, body_pages=list(range(1, 51)), ) @@ -212,12 +239,86 @@ def test_two_contained_tocs_graft_in_order() -> None: assert titles == ["1.2", "1.3"] +def test_classify_contained_when_peer_span_covers_candidate() -> None: + assert ( + classify_toc_relationship( + candidate_span=(32, 50), + host_spans=[(10, 70)], + ) + == "contained" + ) + + +def test_classify_contained_inside_tight_peer_span() -> None: + assert ( + classify_toc_relationship( + candidate_span=(32, 40), + host_spans=[(30, 45), (10, 70)], + ) + == "contained" + ) + + +def test_classify_parallel_when_no_peer_covers_span() -> None: + assert ( + classify_toc_relationship( + candidate_span=(95, 140), + host_spans=[(10, 70)], + ) + == "parallel" + ) + + +def test_classify_parallel_without_host_spans() -> None: + assert ( + classify_toc_relationship( + candidate_span=(20, 22), + host_spans=[], + ) + == "parallel" + ) + + +def test_classify_unresolvable_when_candidate_span_missing() -> None: + assert ( + classify_toc_relationship( + candidate_span=None, + host_spans=[(10, 70)], + ) + == "unresolvable" + ) + + +def test_classify_parallel_for_chapter_mini_tocs_without_extrapolation() -> None: + """Earliest chapter TOC evidenced [6,8] must not swallow later chapters.""" + assert ( + classify_toc_relationship( + candidate_span=(12, 24), + host_spans=[(6, 8)], + ) + == "parallel" + ) + + +def test_find_tightest_host_prefers_smaller_span() -> None: + from app.services.document_agent.structure.toc_anchoring import ( + find_tightest_containing_host, + ) + + assert ( + find_tightest_containing_host( + (32, 40), + [("wide", (10, 70)), ("tight", (30, 45))], + ) + == "tight" + ) + + def _ctx(*, page_count: int) -> ToolContext: return ToolContext( pdf_path="/tmp/doc.pdf", job_id="job-graft", - blackboard=AgentBlackboard(page_count=page_count), - budget=BudgetTracker(plan_budget=50_000, visual_budget=80_000), + blackboard=ProfileBlackboard(page_count=page_count), trace=None, settings={}, ) @@ -231,7 +332,7 @@ def _anchor(pages: dict[tuple[str, ...], int]) -> SkeletonAnchor: null_page_report=[], bulk_count=len(pages), pruned_count=0, - locate_agent="offset_guided_bulk", + locate_method="offset_guided_bulk", ) @@ -259,26 +360,21 @@ def fake_anchor_hierarchy(**kwargs): captured["body_pages"] = kwargs["body_pages"] return [primary], _anchor({("Ch1",): 2}) - anchoring_globals = run_toc_anchoring.__globals__ with ( patch( - "app.services.document_agent.agents.calibration.orchestrator.anchor_hierarchy", + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", side_effect=fake_anchor_hierarchy, ), patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", return_value=object(), ), patch( - "app.services.document_agent.agents.calibration.procedure.pick_primary_offset", + "app.services.document_agent.calibration.procedure.pick_primary_offset", return_value=0, ), - patch.dict( - anchoring_globals, - {"classify_toc_relationship": lambda **_kwargs: "parallel"}, - ), patch( - "app.services.document_agent.agents.calibration.procedure.finalize_calibration_result", + "app.services.document_agent.calibration.procedure.finalize_calibration_result", return_value=([pending], _anchor({("App",): 22}), True), ), ): @@ -297,7 +393,10 @@ def test_profile_grafts_contained_and_keeps_original_pending() -> None: { "toc_range": [1, 1], "toc_range_unit": "page", - "toc_with_level": [{"heading": "第一章", "level": 1, "page_number": 10}], + "toc_with_level": [ + {"heading": "第一章", "level": 1, "page_number": 10}, + {"heading": "第二章", "level": 1, "page_number": 30}, + ], }, { "toc_range": [20, 21], @@ -310,7 +409,10 @@ def test_profile_grafts_contained_and_keeps_original_pending() -> None: ] ctx.blackboard.toc_result = TocResult(method="vlm_batch", toc_pages=[1, 20, 21]) ctx.blackboard.page_full_text_cache = {page: "body" for page in range(1, 51)} - primary = TitleNode(title="第一章", level=1, printed_page=10) + primary = [ + TitleNode(title="第一章", level=1, printed_page=10), + TitleNode(title="第二章", level=1, printed_page=30), + ] contained = TitleNode( title="第一章", level=1, @@ -318,26 +420,24 @@ def test_profile_grafts_contained_and_keeps_original_pending() -> None: children=[TitleNode(title="1.2", level=2, printed_page=12)], ) - anchoring_globals = run_toc_anchoring.__globals__ with ( patch( - "app.services.document_agent.agents.calibration.orchestrator.anchor_hierarchy", - return_value=([primary], _anchor({("第一章",): 10})), + "app.services.document_agent.calibration.orchestrator.anchor_hierarchy", + return_value=( + primary, + _anchor({("第一章",): 10, ("第二章",): 30}), + ), ), patch( - "app.services.document_agent.agents.calibration.service.calibrate_offset", + "app.services.document_agent.calibration.service.calibrate_offset", return_value=object(), ), patch( - "app.services.document_agent.agents.calibration.procedure.pick_primary_offset", + "app.services.document_agent.calibration.procedure.pick_primary_offset", return_value=0, ), - patch.dict( - anchoring_globals, - {"classify_toc_relationship": lambda **_kwargs: "contained"}, - ), patch( - "app.services.document_agent.agents.calibration.procedure.finalize_calibration_result", + "app.services.document_agent.calibration.procedure.finalize_calibration_result", return_value=( [contained], _anchor({("第一章",): 10, ("第一章", "1.2"): 12}), @@ -350,6 +450,8 @@ def test_profile_grafts_contained_and_keeps_original_pending() -> None: assert ctx.blackboard.skeleton_nodes[0]["title"] == "第一章" assert ctx.blackboard.skeleton_nodes[0]["children"][0]["title"] == "1.2" record = ctx.blackboard.pending_skeleton_anchors[0] + assert record["relationship"] == "contained" + assert record["host"] == "root" assert record["grafted"] is True assert record["nodes"][0]["title"] == "第一章" assert "第一章 / 1.2" in ctx.blackboard.skeleton_anchor["match_overrides"] @@ -387,7 +489,7 @@ def _anatomy( page_count=page_count, page_features=[_feature(page) for page in range(1, page_count + 1)], page_labels=[ - PageLabel(page=page, kind="normal", confidence=1.0) + PageLabel(page=page, kind="normal") for page in range(1, page_count + 1) ], toc_result=TocResult(method="vlm_batch", toc_pages=toc_pages), @@ -402,12 +504,10 @@ def _anatomy( page_offset=0, anchor_type="forced_max_size", anchor_evidence="test", - confidence=1.0, ) ], ), toc_hierarchies=hierarchies, - toc_page_offset=0, skeleton_anchor=serialize_skeleton_anchor(_anchor(overrides)), skeleton_nodes=[serialize_title_node(node) for node in nodes], pending_skeleton_anchors=pending_records, diff --git a/apps/worker/tests/contract/test_toc_link_attach_wiring_contract.py b/apps/worker/tests/contract/test_toc_link_attach_wiring_contract.py deleted file mode 100644 index 6ed378141..000000000 --- a/apps/worker/tests/contract/test_toc_link_attach_wiring_contract.py +++ /dev/null @@ -1,193 +0,0 @@ -"""PROFILE attaches TOC-page links before calibration.""" - -from __future__ import annotations - -import os -from types import SimpleNamespace -from unittest.mock import patch - -os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") -os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") -os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") -os.environ.setdefault("S3_ACCESS_KEY_ID", "test") -os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") -os.environ.setdefault("S3_TEMP_PATH", "/tmp") - -from app.services.document_agent.coordinator import ProfileCoordinator -from app.services.document_agent.manifest import ToolResult - - -def _hierarchy() -> list[dict[str, object]]: - return [ - { - "toc_range": [2, 5], - "toc_with_level": [ - {"heading": "Ch1", "level": 1, "page_number": 2}, - ], - } - ] - - -def _coordinator() -> ProfileCoordinator: - coordinator = ProfileCoordinator(pdf_path="/tmp/doc.pdf", job_id="job-link-order") - coordinator.blackboard.toc_hierarchies = _hierarchy() - return coordinator - - -def test_profile_attaches_toc_links_before_anchoring() -> None: - coordinator = _coordinator() - seen: dict[str, object] = {} - globals_ = ProfileCoordinator._run_toc_extraction_pipeline.__globals__ - - def fake_enrich(*, pdf_path: str, toc_hierarchies: list[dict[str, object]]): - assert pdf_path == "/tmp/doc.pdf" - attached = [ - { - **toc_hierarchies[0], - "toc_with_level": [ - { - **toc_hierarchies[0]["toc_with_level"][0], # type: ignore[index] - "link": {"physical_page": 8}, - } - ], - } - ] - return attached, SimpleNamespace( - entries_matched=1, - entries_total=1, - skipped_no_links=False, - ) - - def fake_anchor(ctx) -> None: - entry = ctx.blackboard.toc_hierarchies[0]["toc_with_level"][0] - seen["physical_page"] = (entry.get("link") or {}).get("physical_page") - - with ( - patch.object( - ProfileCoordinator, - "_dispatch_profile_tool", - return_value=ToolResult(status="ok"), - ), - patch.dict( - globals_, - { - "enrich_toc_hierarchies_with_links": fake_enrich, - "run_toc_anchoring": fake_anchor, - }, - ), - ): - coordinator._run_toc_extraction_pipeline() - - assert seen["physical_page"] == 8 - assert ( - coordinator.blackboard.toc_hierarchies[0]["toc_with_level"][0]["link"][ - "physical_page" - ] - == 8 - ) - - -def test_link_attach_failure_keeps_hierarchies_and_still_anchors() -> None: - coordinator = _coordinator() - seen = {"anchored": False} - globals_ = ProfileCoordinator._run_toc_extraction_pipeline.__globals__ - - def fake_anchor(ctx) -> None: - seen["anchored"] = True - entry = ctx.blackboard.toc_hierarchies[0]["toc_with_level"][0] - assert entry.get("link") is None - - def boom_enrich(**_kwargs): - raise RuntimeError("pymupdf failed") - - with ( - patch.object( - ProfileCoordinator, - "_dispatch_profile_tool", - return_value=ToolResult(status="ok"), - ), - patch.dict( - globals_, - { - "enrich_toc_hierarchies_with_links": boom_enrich, - "run_toc_anchoring": fake_anchor, - }, - ), - ): - coordinator._run_toc_extraction_pipeline() - - assert seen["anchored"] is True - assert coordinator.blackboard.toc_hierarchies == _hierarchy() - - -def test_skip_toc_anchoring_stops_after_link_attach() -> None: - coordinator = _coordinator() - coordinator.ctx.settings["skip_toc_anchoring"] = True - seen = {"anchored": False} - globals_ = ProfileCoordinator._run_toc_extraction_pipeline.__globals__ - - def fake_enrich(*, pdf_path: str, toc_hierarchies: list[dict[str, object]]): - return toc_hierarchies, SimpleNamespace( - entries_matched=0, - entries_total=1, - skipped_no_links=True, - ) - - def fake_anchor(_ctx) -> None: - seen["anchored"] = True - - with ( - patch.object( - ProfileCoordinator, - "_dispatch_profile_tool", - return_value=ToolResult(status="ok"), - ), - patch.dict( - globals_, - { - "enrich_toc_hierarchies_with_links": fake_enrich, - "run_toc_anchoring": fake_anchor, - }, - ), - ): - coordinator._run_toc_extraction_pipeline() - - assert seen["anchored"] is False - assert coordinator.blackboard.skeleton_anchor is None - assert coordinator.blackboard.skeleton_nodes is None - assert coordinator.blackboard.toc_page_offset is None - - -def test_stop_after_asset_probe_skips_toc() -> None: - coordinator = ProfileCoordinator(pdf_path="/tmp/doc.pdf", job_id="job-stage0") - coordinator.ctx.settings["stop_after_asset_probe"] = True - seen = {"toc": False, "assets": False} - - profile = SimpleNamespace( - category="spec", - routing_category="generic", - is_scanned=False, - ) - - def fake_toc(self, *, strict: bool) -> None: - seen["toc"] = True - - def fake_assets(self) -> None: - seen["assets"] = True - - with ( - patch.object(ProfileCoordinator, "_run_bootstrap", return_value=None), - patch.object( - ProfileCoordinator, - "_propose_profile", - return_value=(profile, None, ToolResult(status="ok")), - ), - patch.object(ProfileCoordinator, "_run_text_scan", return_value=None), - patch.object(ProfileCoordinator, "_ensure_toc_profile", fake_toc), - patch.object(ProfileCoordinator, "_ensure_asset_probe", fake_assets), - ): - out = coordinator._run_coarse() - - assert out is profile - assert seen["assets"] is True - assert seen["toc"] is False diff --git a/apps/worker/tests/contract/test_toc_link_match_contract.py b/apps/worker/tests/contract/test_toc_link_match_contract.py deleted file mode 100644 index 54b35c78c..000000000 --- a/apps/worker/tests/contract/test_toc_link_match_contract.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Contract tests for TOC heading → link containment matching.""" - -from __future__ import annotations - -import os - -os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") -os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") -os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") -os.environ.setdefault("S3_ACCESS_KEY_ID", "test") -os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") -os.environ.setdefault("S3_TEMP_PATH", "/tmp") - -from app.services.document_agent.structure.toc_link_enrichment import ( - TocPageLink, - _link_dest_physical_page, - match_toc_entries_to_links, -) - - -def _link(anchor: str, dest: int, *, toc_page: int = 2) -> TocPageLink: - return TocPageLink( - toc_page=toc_page, - dest_physical_page=dest, - anchor_text=anchor, - kind=4, - ) - - -def test_get_links_dest_page_is_already_one_based() -> None: - """``get_links()['page']`` must not be shifted with +1.""" - assert _link_dest_physical_page(6) == 6 - assert _link_dest_physical_page("6") == 6 - assert _link_dest_physical_page("7") == 7 - - -def test_match_exact_one_hit_attaches_physical_page() -> None: - entries = [ - {"heading": " 1.投标人营业执照扫描件; ", "level": 2, "page_number": 2}, - {"heading": "二、施工组织设计", "level": 1, "page_number": 26}, - ] - links = [ - _link("1.投标人营业执照扫描件;...............................", 7), - _link("二、施工组织设计.............................................", 31), - _link("无关导航链接", 99), - ] - - enriched, matched = match_toc_entries_to_links(entries, links) - - assert matched == 2 - assert enriched[0]["link"] == {"physical_page": 7} - assert enriched[1]["link"] == {"physical_page": 31} - assert enriched[0]["heading"] == " 1.投标人营业执照扫描件; " - assert enriched[0]["page_number"] == 2 - - -def test_match_zero_or_many_hits_leaves_entry_unmatched() -> None: - entries = [ - {"heading": "一、资格复审资料", "level": 1, "page_number": 1}, - {"heading": "共用标题", "level": 2, "page_number": 3}, - ] - links = [ - _link("共用标题..............2", 10), - _link("共用标题..............9", 20), - ] - - enriched, matched = match_toc_entries_to_links(entries, links) - - assert matched == 0 - assert "link" not in enriched[0] - assert "link" not in enriched[1] - - -def test_match_processes_vlm_order_once_each() -> None: - entries = [ - {"heading": "第一章", "level": 1, "page_number": 1}, - {"heading": "第二章", "level": 1, "page_number": 5}, - ] - links = [ - _link("第二章........5", 15), - _link("第一章........1", 11), - ] - - enriched, matched = match_toc_entries_to_links(entries, links) - - assert matched == 2 - assert [e["heading"] for e in enriched] == ["第一章", "第二章"] - assert enriched[0]["link"]["physical_page"] == 11 - assert enriched[1]["link"]["physical_page"] == 15 - - -def test_match_strips_stale_link_when_unmatched() -> None: - entries = [ - { - "heading": "无匹配", - "level": 1, - "page_number": 1, - "link": {"physical_page": 99}, - }, - ] - enriched, matched = match_toc_entries_to_links(entries, [_link("别的标题..1", 2)]) - - assert matched == 0 - assert "link" not in enriched[0] diff --git a/apps/worker/tests/contract/test_toc_phase2_region_concurrency_contract.py b/apps/worker/tests/contract/test_toc_phase2_region_concurrency_contract.py index a068b0b88..2d2d89d86 100644 --- a/apps/worker/tests/contract/test_toc_phase2_region_concurrency_contract.py +++ b/apps/worker/tests/contract/test_toc_phase2_region_concurrency_contract.py @@ -43,24 +43,30 @@ def _batch_result( toc_pages: list[int], non_toc_pages: list[int], entries: list[dict[str, Any]] | None = None, + page_results: list[BatchPageResult] | None = None, ) -> BatchTocResult: - page_results: list[BatchPageResult] = [ - BatchPageResult(page=page, is_toc=True, entries=[]) for page in toc_pages - ] - page_results.extend( - BatchPageResult(page=page, is_toc=False, entries=[]) for page in non_toc_pages - ) - if page_results and page_results[-1].is_toc: - page_results[-1] = BatchPageResult( - page=page_results[-1].page, - is_toc=False, - entries=[], + entry_list = list(entries or []) + if page_results is None: + # Callers pass ordered partitions (TOC prefix then non-TOC suffix). + page_results = [] + for index, page in enumerate(toc_pages): + page_entries = entry_list if index == 0 else [] + page_results.append( + BatchPageResult(page=page, is_toc=True, entries=page_entries) + ) + page_results.extend( + BatchPageResult(page=page, is_toc=False, entries=[]) + for page in non_toc_pages ) + all_entries = list(entry_list) + if not all_entries: + for page_result in page_results: + all_entries.extend(list(page_result.entries or [])) return BatchTocResult( page_results=page_results, toc_pages=list(toc_pages), non_toc_pages=list(non_toc_pages), - all_entries=list(entries or []), + all_entries=all_entries, meta={"ok": True}, ) @@ -274,3 +280,193 @@ def _fake_batch( assert render_max == 1 assert spawn_count == 3 # one batch spawn per anchor window assert vlm_max >= 2 + + +def test_contiguous_toc_prefix_stops_at_first_non_toc() -> None: + page_results = [ + BatchPageResult( + page=40, + is_toc=True, + entries=[{"title": "3.1", "page_number": 1, "level": 1}], + ), + BatchPageResult(page=41, is_toc=False, entries=[]), + BatchPageResult(page=42, is_toc=False, entries=[]), + BatchPageResult(page=43, is_toc=False, entries=[]), + BatchPageResult( + page=44, + is_toc=True, + entries=[{"title": "4.1", "page_number": 1, "level": 1}], + ), + ] + kept_pages, kept_entries = toc_tool._contiguous_toc_prefix(page_results) # noqa: SLF001 + assert kept_pages == [40] + assert kept_entries == [{"title": "3.1", "page_number": 1, "level": 1}] + + +def test_should_expand_requires_empty_non_toc_and_full_window_end() -> None: + expand = toc_tool._should_expand_toc_window # noqa: SLF001 + assert expand( + batch_start=40, + non_toc_pages=[], + kept_toc_pages=[40, 41, 42, 43, 44], + ) + # Mid-window break: last TOC may still equal full-window end. + assert not expand( + batch_start=40, + non_toc_pages=[41, 42, 43], + kept_toc_pages=[40], + ) + # Short end-of-doc window: all TOC but not a full step. + assert not expand( + batch_start=98, + non_toc_pages=[], + kept_toc_pages=[98, 99, 100], + ) + + +def test_extract_region_drops_post_break_toc_and_does_not_expand( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Ch3-style: TOC then body then next-chapter TOC in one 5-page window.""" + confirmed = _anchors(tmp_path, [40]) + vlm_rounds: list[list[int]] = [] + + def _fake_batch( + *, + page_pngs: list[tuple[int, str]], + model: str, + previous_entries: list[dict[str, Any]] | None = None, + ) -> BatchTocResult: + pages = [page for page, _ in page_pngs] + vlm_rounds.append(pages) + assert pages == [40, 41, 42, 43, 44] + return _batch_result( + toc_pages=[40, 44], + non_toc_pages=[41, 42, 43], + page_results=[ + BatchPageResult( + page=40, + is_toc=True, + entries=[ + {"title": "3.1 INTRODUCTION", "page_number": 1, "level": 1} + ], + ), + BatchPageResult(page=41, is_toc=False, entries=[]), + BatchPageResult(page=42, is_toc=False, entries=[]), + BatchPageResult(page=43, is_toc=False, entries=[]), + BatchPageResult( + page=44, + is_toc=True, + entries=[ + {"title": "4.1 INTRODUCTION", "page_number": 1, "level": 1} + ], + ), + ], + ) + + monkeypatch.setattr(toc_tool, "run_in_child_process", _fake_batch_render) + monkeypatch.setattr( + "app.services.document_agent.tools.vlm_toc_extractor.vlm_extract_toc_batch", + _fake_batch, + ) + + regions = toc_tool._extract_regions_for_confirmed_anchors( # noqa: SLF001 + confirmed, + pdf_path="/tmp/doc.pdf", + page_count=100, + output_dir=str(tmp_path / "toc_pages"), + dpi=72, + model="fake-vlm", + ) + + assert len(vlm_rounds) == 1 + assert regions[0].toc_pages == [40] + assert regions[0].entries == [ + {"title": "3.1 INTRODUCTION", "page_number": 1, "level": 1} + ] + assert regions[0].hierarchies + assert regions[0].hierarchies[0]["toc_range"] == [40, 40] + assert regions[0].batch_trace[0]["kept_toc_pages"] == [40] + assert regions[0].batch_trace[0]["expanded"] is False + + +def test_extract_region_expands_only_on_full_unbroken_toc_window( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + confirmed = _anchors(tmp_path, [10]) + vlm_rounds: list[list[int]] = [] + + def _fake_batch( + *, + page_pngs: list[tuple[int, str]], + model: str, + previous_entries: list[dict[str, Any]] | None = None, + ) -> BatchTocResult: + pages = [page for page, _ in page_pngs] + vlm_rounds.append(pages) + if pages[0] == 10: + assert previous_entries in (None, []) + return _batch_result( + toc_pages=pages, + non_toc_pages=[], + page_results=[ + BatchPageResult( + page=page, + is_toc=True, + entries=[ + { + "title": f"Entry {page}", + "page_number": page, + "level": 1, + } + ], + ) + for page in pages + ], + ) + return _batch_result( + toc_pages=[pages[0]], + non_toc_pages=pages[1:], + page_results=[ + BatchPageResult( + page=pages[0], + is_toc=True, + entries=[ + { + "title": f"Entry {pages[0]}", + "page_number": pages[0], + "level": 1, + } + ], + ), + *[ + BatchPageResult(page=page, is_toc=False, entries=[]) + for page in pages[1:] + ], + ], + ) + + monkeypatch.setattr(toc_tool, "run_in_child_process", _fake_batch_render) + monkeypatch.setattr( + "app.services.document_agent.tools.vlm_toc_extractor.vlm_extract_toc_batch", + _fake_batch, + ) + + regions = toc_tool._extract_regions_for_confirmed_anchors( # noqa: SLF001 + confirmed, + pdf_path="/tmp/doc.pdf", + page_count=100, + output_dir=str(tmp_path / "toc_pages"), + dpi=72, + model="fake-vlm", + ) + + assert vlm_rounds == [ + [10, 11, 12, 13, 14], + [15, 16, 17, 18, 19], + ] + assert regions[0].toc_pages == [10, 11, 12, 13, 14, 15] + assert regions[0].batch_trace[0]["expanded"] is True + assert regions[0].batch_trace[1]["expanded"] is False diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py new file mode 100644 index 000000000..031266c02 --- /dev/null +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -0,0 +1,38 @@ +"""Smoke: registered tools include outline/links/inspect; direct call still works.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +import app.services.document_agent.tools as _tools # noqa: F401 +from app.services.document_agent.registry import REGISTRY +from app.services.document_agent.tools.inspect_pages import inspect_pages + + +def test_probe_and_inspect_registered() -> None: + for name in ( + "probe.outline", + "probe.links", + "judge.toc_source", + "inspect.pages", + "ocr.pages", + "grep.text", + ): + assert REGISTRY.get(name) is not None, name + + +def test_inspect_pages_handler_is_same_callable() -> None: + spec = REGISTRY.get("inspect.pages") + assert spec is not None + assert spec.handler is inspect_pages + + +def test_openai_specs_removed() -> None: + assert not hasattr(REGISTRY, "openai_specs") diff --git a/apps/worker/tests/unit/test_planner_sample_pages.py b/apps/worker/tests/unit/test_coarse_profile_sample_pages.py similarity index 90% rename from apps/worker/tests/unit/test_planner_sample_pages.py rename to apps/worker/tests/unit/test_coarse_profile_sample_pages.py index 31f9eadf9..8d2e7d9ad 100644 --- a/apps/worker/tests/unit/test_planner_sample_pages.py +++ b/apps/worker/tests/unit/test_coarse_profile_sample_pages.py @@ -1,10 +1,10 @@ -"""Coarse planner page sampling.""" +"""Coarse profile page sampling.""" from __future__ import annotations import random -from app.services.document_agent.planner.planner import _sample_pages +from app.services.document_agent.coarse_profile.classifier import _sample_pages def test_sample_pages_with_extrema_keeps_extrema_first() -> None: diff --git a/packages/shared-python/shared/core/config/storage.py b/packages/shared-python/shared/core/config/storage.py index 0fc8571eb..e04d69d64 100644 --- a/packages/shared-python/shared/core/config/storage.py +++ b/packages/shared-python/shared/core/config/storage.py @@ -80,13 +80,14 @@ class StorageConfig(BaseModel): "Documents exceeding this are rejected with a contact-support message.", ) PDF_PROFILE_TOC_ENABLED: bool = Field( - default=False, + default=True, description=( - "Enable page-owned PDF TOC profiling for the CHUNK track during " - "parser-entry DOC_PROFILE. When disabled, chunk-track PDF parsing " - "treats documents as no-TOC and does not fall back to Markdown TOC " - "detection. NOTE: the page-memory track always forces TOC profiling " - "on regardless of this flag, since its sections are TOC-anchored." + "Default-on PROFILE TOC find/extract/calibration for PDF parsing " + "(chunk and page_memory tracks share this pipeline). When True, " + "missing TOC or calibration failure degrades to no-TOC downstream. " + "Set False only as an emergency kill switch for the chunk track. " + "NOTE: page_memory always forces TOC profiling on regardless of " + "this flag, since its sections are TOC-anchored." ), ) MINERU_SHARD_CONCURRENCY: int = Field( diff --git a/packages/shared-python/shared/services/ai/prompt_service.py b/packages/shared-python/shared/services/ai/prompt_service.py index f5152cab7..808b816c9 100755 --- a/packages/shared-python/shared/services/ai/prompt_service.py +++ b/packages/shared-python/shared/services/ai/prompt_service.py @@ -357,58 +357,6 @@ def build_prompt(task, texts, query, **kwargs): - Do not add any explanations, comments, control characters, or descriptive texts. """ - # ==================== Merge-Group Pre-pass Prompt ==================== - - elif task == "eval-merge-groups": - # Focused single-question prompt: ONLY decides merge vs. keep for - # groups of consecutive heading candidates (no body text between them). - # Does NOT assign hierarchy levels — that is left to the main LLM call. - temperature = 0 - top_p = 0.01 - max_tokens = kwargs["paras"].get("max_tokens", 800) - prompt = f""" - You are a PDF heading reconstruction expert. - - A PDF renderer sometimes splits a single long heading title across multiple - consecutive lines. You will receive a numbered list of groups. Each group - contains 2–6 consecutive heading candidate lines from a PDF with NO body - text between them. - - Your ONLY task: for each group, decide whether the lines should be MERGED - into one single heading, or kept as SEPARATE headings in a parent-child - relationship. - - **MERGE when ALL hold:** - 1. Reading the lines in sequence produces ONE grammatically complete, - natural-sounding title — no missing words, no awkward break. - 2. The first line alone is grammatically INCOMPLETE as a standalone title - (e.g. ends with a possessive "'s", a preposition "of / for / and", - a conjunction, or is otherwise a dangling fragment). - 3. No semantic gap: every subsequent line is a direct lexical extension - of the first, not a new sub-topic. - - **KEEP SEPARATE when ANY hold:** - - The first line is already a complete, self-contained title on its own. - - Subsequent lines introduce a different topic or finer sub-topic. - - Lines form a clear parent-heading → child-heading sequence. - - **Any subsequent line begins with a numeric or ordinal prefix** such as - `01`, `1.`, `(1)`, `①`, `一、`, `第一` — these are numbered sub-items, - never continuation fragments of the preceding heading. - - **Generic linguistic signals that indicate MERGE:** - - Line ends with a possessive ("Company's", "Board's") — demands a noun phrase. - - Line ends with a preposition ("of", "for", "under", "and") — phrase is incomplete. - - Line ends mid-adjective or mid-noun phrase that continues on the next line. - - Groups to evaluate: - {texts} - - Output a JSON array — one object per group, in the SAME ORDER as the input: - [{{"group": 1, "merge": true}}, {{"group": 2, "merge": false}}, ...] - - Output ONLY valid JSON. No markdown fences, no explanations. - """ - # ==================== TOC Heading Evaluation Prompts ==================== elif task == "eval-toc-headings": diff --git a/packages/shared-python/shared/services/ai/summary/engine.py b/packages/shared-python/shared/services/ai/summary/engine.py index 141520f6f..2066ab795 100644 --- a/packages/shared-python/shared/services/ai/summary/engine.py +++ b/packages/shared-python/shared/services/ai/summary/engine.py @@ -257,6 +257,10 @@ def summarize( max_keywords: int = 5, model: str | None = None, usage_task: str | None = None, + # TODO(parse-budget-cleanup): no live caller passes a non-None budget after + # PROFILE BudgetTracker removal. Drop budget/budget_pool/budget_stage once + # remaining formats stop needing this duck-typed hook, or redirect any + # future limit to token_tracking instead. budget: Any | None = None, budget_pool: str = "visual", budget_stage: str | None = None, @@ -276,7 +280,8 @@ def summarize( Page or asset image(s). Required for ``page``/``asset`` modes that render from an image; ignored for plain ``text``. budget: - Optional ``BudgetTracker``. Visual calls reserve from ``budget_stage``. + Optional external reservation ledger. Visual calls reserve from + ``budget_stage``. prompt_task / prompt_paras: Override the prompt used for the image-based page path. Lets a bounded node summary (``page-memory-node-summary`` with ``node_title`` / diff --git a/packages/shared-python/shared/services/ai/token_costing.py b/packages/shared-python/shared/services/ai/token_costing.py index 36e7e1363..d8ffe18b9 100644 --- a/packages/shared-python/shared/services/ai/token_costing.py +++ b/packages/shared-python/shared/services/ai/token_costing.py @@ -13,14 +13,23 @@ from shared.core.config import settings DEFAULT_TOKEN_PRICING_TABLE: dict[str, dict[str, Any]] = { + # DeepSeek official (peak/off-peak since 2026-08-16 16:00 UTC). + # Defaults use OFF-PEAK; peak rates are 2x (01:00-04:00 & 06:00-10:00 UTC). "deepseek-v4-flash": { "currency": "USD", "unit": "per_1m_tokens", - "input_per_1m": 0.14, - "cached_input_per_1m": 0.0028, - "output_per_1m": 0.28, - "source": "DeepSeek official pricing", - "effective_date": "2026-06-11", + "input_per_1m": 0.22, + "cached_input_per_1m": 0.007, + "output_per_1m": 0.66, + "peak_input_per_1m": 0.44, + "peak_cached_input_per_1m": 0.014, + "peak_output_per_1m": 1.32, + "source": ( + "DeepSeek official pricing " + "(https://api-docs.deepseek.com/quick_start/pricing); " + "defaults are off-peak cache-miss rates" + ), + "effective_date": "2026-08-17", }, "deepseek-chat": { "alias_of": "deepseek-v4-flash", @@ -28,13 +37,19 @@ "deepseek-reasoner": { "alias_of": "deepseek-v4-flash", }, + # China mainland DashScope (ALI_URL=dashscope.aliyuncs.com), <=256K tier. + # Official CNY is ¥1.2 / ¥7.2 per 1M; Bailian USD display is $0.165 / $0.99. "qwen3.6-flash": { "currency": "USD", "unit": "per_1m_tokens", - "input_per_1m": 0.25, - "output_per_1m": 1.50, - "source": "Qwen Cloud official pricing, <=256K input tier", - "effective_date": "2026-06-11", + "input_per_1m": 0.165, + "output_per_1m": 0.99, + "source": ( + "Alibaba Bailian China mainland qwen3.6-flash <=256K " + "(¥1.2/¥7.2 per 1M; USD list $0.165/$0.99). " + "International endpoint is $0.25/$1.50" + ), + "effective_date": "2026-08-17", }, } diff --git a/packages/shared-python/shared/services/ai/token_tracking.py b/packages/shared-python/shared/services/ai/token_tracking.py index 99da9108b..a0f61d1eb 100644 --- a/packages/shared-python/shared/services/ai/token_tracking.py +++ b/packages/shared-python/shared/services/ai/token_tracking.py @@ -14,6 +14,8 @@ from __future__ import annotations import threading +from contextlib import contextmanager +from collections.abc import Iterator from typing import Any _trackers: dict[int, dict[str, Any]] = {} @@ -24,6 +26,11 @@ # their root. We walk the greenlet parent chain to find the id. _root_ids: dict[int, int] = {} +# TODO(parse-total-token-limit): Add one optional parse-wide token limit here, +# defaulting to unlimited. Enforcement must reserve before provider calls and +# commit/refund atomically so concurrent greenlets and native threads share the +# same limit without reintroducing stage-specific ledgers. + def _current_greenlet_id() -> int: try: @@ -88,6 +95,32 @@ def get_current_token_tracker() -> dict[str, Any] | None: return _trackers.get(root) +def get_current_token_tracker_root_id() -> int | None: + """Return the active parse tracker root id for child-context propagation.""" + return _find_root_id() + + +@contextmanager +def bind_token_tracker(root_id: int | None) -> Iterator[None]: + """Temporarily bind the current native thread/greenlet to a parse tracker.""" + if root_id is None: + yield + return + + gid = _current_greenlet_id() + with _lock: + previous = _root_ids.get(gid) + _root_ids[gid] = root_id + try: + yield + finally: + with _lock: + if previous is None: + _root_ids.pop(gid, None) + else: + _root_ids[gid] = previous + + def cleanup_token_tracker() -> None: """Remove the tracker for the current greenlet. Call after parsing.""" gid = _current_greenlet_id() diff --git a/packages/shared-python/shared/tests/test_token_tracking.py b/packages/shared-python/shared/tests/test_token_tracking.py new file mode 100644 index 000000000..9e126fe92 --- /dev/null +++ b/packages/shared-python/shared/tests/test_token_tracking.py @@ -0,0 +1,41 @@ +from concurrent.futures import ThreadPoolExecutor + +from shared.services.ai.token_tracking import ( + bind_token_tracker, + cleanup_token_tracker, + get_current_token_tracker, + get_current_token_tracker_root_id, + init_token_tracker, + record_tokens, +) + + +def test_native_thread_usage_is_recorded_on_parse_tracker() -> None: + tracker = init_token_tracker() + root_id = get_current_token_tracker_root_id() + + def record_from_thread() -> None: + with bind_token_tracker(root_id): + record_tokens( + { + "prompt_tokens": 11, + "completion_tokens": 7, + "total_tokens": 18, + }, + model="test-model", + task="parser.test.thread", + ) + + try: + with ThreadPoolExecutor(max_workers=1) as executor: + executor.submit(record_from_thread).result() + + assert get_current_token_tracker() is tracker + assert tracker["prompt_tokens"] == 11 + assert tracker["completion_tokens"] == 7 + assert tracker["total_tokens"] == 18 + assert tracker["calls"] == 1 + assert tracker["by_task"]["parser.test.thread"]["total_tokens"] == 18 + assert tracker["by_model"]["test-model"]["total_tokens"] == 18 + finally: + cleanup_token_tracker() diff --git a/uv.lock b/uv.lock index ff85b4332..77a0eee55 100644 --- a/uv.lock +++ b/uv.lock @@ -805,15 +805,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829, upload-time = "2026-04-08T01:57:48.874Z" }, ] -[[package]] -name = "defusedxml" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, -] - [[package]] name = "deprecated" version = "1.3.1" @@ -1580,14 +1571,12 @@ dependencies = [ { name = "logfire", extra = ["celery", "fastapi", "httpx", "sqlalchemy"] }, { name = "lxml" }, { name = "markdownify" }, - { name = "markitdown" }, { name = "numpy" }, { name = "openai" }, { name = "openpyxl" }, { name = "oss2" }, { name = "pandas" }, { name = "pillow" }, - { name = "pptx2md" }, { name = "psycogreen" }, { name = "pymupdf" }, { name = "pymupdf4llm" }, @@ -1619,14 +1608,12 @@ requires-dist = [ { name = "logfire", extras = ["celery", "fastapi", "httpx", "sqlalchemy"], specifier = ">=4.25.0" }, { name = "lxml", specifier = "==6.1.0" }, { name = "markdownify", specifier = "==1.2.2" }, - { name = "markitdown", specifier = "==0.1.2" }, { name = "numpy", specifier = "==2.2.6" }, { name = "openai", specifier = "==1.93.3" }, { name = "openpyxl", specifier = "==3.1.2" }, { name = "oss2", specifier = ">=2.18.0" }, { name = "pandas", specifier = "==2.3.1" }, { name = "pillow", specifier = "==12.2.0" }, - { name = "pptx2md", specifier = "==2.0.6" }, { name = "psycogreen", specifier = ">=1.0.2" }, { name = "pymupdf", specifier = "==1.27.2" }, { name = "pymupdf4llm", specifier = "==1.27.2.1" }, @@ -1882,24 +1869,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/3ee09a5b60cb44c4f2fbc1c9015cfd6ff5afc08f991cab295d3024dcbf2d/lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace", size = 3508860, upload-time = "2026-04-18T04:32:48.619Z" }, ] -[[package]] -name = "magika" -version = "0.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "numpy" }, - { name = "onnxruntime" }, - { name = "python-dotenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/8fdd991142ad3e037179a494b153f463024e5a211ef3ad948b955c26b4de/magika-0.6.2.tar.gz", hash = "sha256:37eb6ae8020f6e68f231bc06052c0a0cbe8e6fa27492db345e8dc867dbceb067", size = 3036634, upload-time = "2025-05-02T14:54:18.88Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/07/4f7748f34279f2852068256992377474f9700b6fbad6735d6be58605178f/magika-0.6.2-py3-none-any.whl", hash = "sha256:5ef72fbc07723029b3684ef81454bc224ac5f60986aa0fc5a28f4456eebcb5b2", size = 2967609, upload-time = "2025-05-02T14:54:09.696Z" }, - { url = "https://files.pythonhosted.org/packages/64/6d/0783af677e601d8a42258f0fbc47663abf435f927e58a8d2928296743099/magika-0.6.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9109309328a1553886c8ff36c2ee9a5e9cfd36893ad81b65bf61a57debdd9d0e", size = 12404787, upload-time = "2025-05-02T14:54:16.963Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ad/42e39748ddc4bbe55c2dc1093ce29079c04d096ac0d844f8ae66178bc3ed/magika-0.6.2-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:57cd1d64897634d15de552bd6b3ae9c6ff6ead9c60d384dc46497c08288e4559", size = 15091089, upload-time = "2025-05-02T14:54:11.59Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1f/28e412d0ccedc068fbccdae6a6233faaa97ec3e5e2ffd242e49655b10064/magika-0.6.2-py3-none-win_amd64.whl", hash = "sha256:711f427a633e0182737dcc2074748004842f870643585813503ff2553b973b9f", size = 12385740, upload-time = "2025-05-02T14:54:14.096Z" }, -] - [[package]] name = "makefun" version = "1.16.0" @@ -1946,23 +1915,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" }, ] -[[package]] -name = "markitdown" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "charset-normalizer" }, - { name = "defusedxml" }, - { name = "magika" }, - { name = "markdownify" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/bd/b7ae7863ee556411fbb6ca19a4a7593ef2b3531d6cd10b979ba386a2dd4d/markitdown-0.1.2.tar.gz", hash = "sha256:85fe108a92bd18f317e75a36cf567a6fa812072612a898abf8c156d5d74c13c4", size = 39361, upload-time = "2025-05-28T17:06:10.423Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/33/d52d06b44c28e0db5c458690a4356e6abbb866f4abc00c0cf4eebb90ca78/markitdown-0.1.2-py3-none-any.whl", hash = "sha256:4881f0768794ffccb52d09dd86498813a6896ba9639b4fc15512817f56ed9d74", size = 57751, upload-time = "2025-05-28T17:06:08.722Z" }, -] - [[package]] name = "markupsafe" version = "3.0.3" @@ -2747,24 +2699,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/c5/0db39bdf91ae2e473ba139568ed24d631e87caac6c175f466d06eedc8747/posthog-7.18.1-py3-none-any.whl", hash = "sha256:54797ae8767911dfd83541f69a9e4fda65e100cb77dc0cf093fd98a3b84916a6", size = 270818, upload-time = "2026-06-10T14:22:26.849Z" }, ] -[[package]] -name = "pptx2md" -version = "2.0.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pillow" }, - { name = "pydantic" }, - { name = "python-pptx" }, - { name = "rapidfuzz" }, - { name = "scipy" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/68/84/023bbef348a8af7efb4e6ce8ca0d70e04af9d2e97b976d7c8fdbb2bf1889/pptx2md-2.0.6.tar.gz", hash = "sha256:2adc052d9fb5e031b0760887eea931e7c78c208b7b2440b10468a3c2874b4e44", size = 15618, upload-time = "2024-12-03T17:41:30.918Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/71/996df5496598f007ce0a5bacf327e6611564c1b8bb2de5f505862b37743c/pptx2md-2.0.6-py3-none-any.whl", hash = "sha256:060937cf4a046544146d117838809c53c10fc7d3f406075f801f112b10a16f04", size = 21804, upload-time = "2024-12-03T17:41:29.102Z" }, -] - [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -3515,85 +3449,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/21/f691fb2613100a62b3fa91e9988c991e9ca5b89ea31c0d3152a3210344f9/rank_bm25-0.2.2-py3-none-any.whl", hash = "sha256:7bd4a95571adadfc271746fa146a4bcfd89c0cf731e49c3d1ad863290adbe8ae", size = 8584, upload-time = "2022-02-16T12:10:50.626Z" }, ] -[[package]] -name = "rapidfuzz" -version = "3.14.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz", hash = "sha256:ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e", size = 57901753, upload-time = "2026-04-07T11:16:31.931Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/f9/3c41a7be8855803f4f6c713b472226a98d31d41869d98f64f4ca790510d6/rapidfuzz-3.14.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e251126d48615e1f02b4a178f2cd0cd4f0332b8a019c01a2e10480f7552554b4", size = 1952372, upload-time = "2026-04-07T11:13:58.32Z" }, - { url = "https://files.pythonhosted.org/packages/9e/89/c2557e37531d03465193bff0ab9de70b468420a807d71a26a65100635459/rapidfuzz-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ab449c9abd0d4e1f8145dce0798a4c822a1a1933d613c764a641bea88b8bdab", size = 1159782, upload-time = "2026-04-07T11:14:00.127Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b2/ffeeb7eca1a897d51b998f4c0ef0281696c3b06abcca4f88f9def708ffe1/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb2829fedd672dd7107267189dabe2bbe07972801d636014417c6861eb89e358", size = 1383677, upload-time = "2026-04-07T11:14:01.696Z" }, - { url = "https://files.pythonhosted.org/packages/6b/d0/4539e42a2d596e068f7738f279638a4a74edd1fbb6f8594e2458058979c6/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d50e5861872935fece391351cbb5ba21d1bced277cf5e1143d207a0a35f1925", size = 3168906, upload-time = "2026-04-07T11:14:03.29Z" }, - { url = "https://files.pythonhosted.org/packages/5e/1c/3ec897eb9d8b05308aa8ef6ae4ed64b088ad521a3f9d8ff469e7e97bc2b0/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:7092a216728f80c960bd6b3807275d1ee318b168986bd5dc523349581d4890b8", size = 1478176, upload-time = "2026-04-07T11:14:04.94Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ba/970c03a12ce20a5399e22afe9f8932fd4cd1265b8a8461d0e63b00eb4eae/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9669753caef7fdc6529f6adcc5883ed98d65976445d9322e7dbdb6b697feee13", size = 2402441, upload-time = "2026-04-07T11:14:07.228Z" }, - { url = "https://files.pythonhosted.org/packages/81/93/61d351cae60c1d0e21ba5ff1a1015ad045539ed215da9d6e302204ed887a/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:823b1b9d9230809d8edcc18872770764bfe8ef4357995e16744047c8ccf0e489", size = 2511628, upload-time = "2026-04-07T11:14:09.234Z" }, - { url = "https://files.pythonhosted.org/packages/87/52/374d2d4f60fd98155142a869323aa221e30868cfa1f15171a0f64070c247/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f0b2af76b7e7060c09e1a0dfa9410eb19369cbe6164509bff2ef94094b54d2b6", size = 4275480, upload-time = "2026-04-07T11:14:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/d8/04/82e7989bc9ec20a15b720a335c5cb6b0724bf6582013898f90a3280cfccd/rapidfuzz-3.14.5-cp311-cp311-win32.whl", hash = "sha256:c5801a89604c65ab4cc9e91b23bc4076d0ca80efd8c976fb63843d7879a85d7f", size = 1725627, upload-time = "2026-04-07T11:14:13.217Z" }, - { url = "https://files.pythonhosted.org/packages/b9/b5/eca8ac5609bc9bcb02bb6ff87fa5983cc92b8772d66a431556ab8a8c178f/rapidfuzz-3.14.5-cp311-cp311-win_amd64.whl", hash = "sha256:d7ca16637c0ede8243f84074044bd0b2335a0341421f8227c85756de2d18c819", size = 1545977, upload-time = "2026-04-07T11:14:14.766Z" }, - { url = "https://files.pythonhosted.org/packages/ca/e1/dbf318de28f65fa2cdd0a9dfbdee380f8199eb83b19259bc4f8592551b4e/rapidfuzz-3.14.5-cp311-cp311-win_arm64.whl", hash = "sha256:8c90cdf8516d9057e502aa6003cea71cf5ec27cc44699ca52412b502a04761bb", size = 816827, upload-time = "2026-04-07T11:14:16.788Z" }, - { url = "https://files.pythonhosted.org/packages/d3/e3/574435c6aafb80254c191ef40d7aca2cb2bb97a095ec9395e9fa59ac307a/rapidfuzz-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0d3378f471ef440473a396ce2f8e97ee12f89a78b495540e0a5617bbfe895638", size = 1944601, upload-time = "2026-04-07T11:14:18.771Z" }, - { url = "https://files.pythonhosted.org/packages/d0/1f/fbad3102a255ecc112ce9a7e779bacab7fd14398217be8868dc9082ba363/rapidfuzz-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e910eebca9fd0eba245c0555e764597e8a0cccb673a92da2dc2397050725f48", size = 1164293, upload-time = "2026-04-07T11:14:20.534Z" }, - { url = "https://files.pythonhosted.org/packages/88/37/a3eb7ff6121ed3a5f199a8c38cc86c8e481816f879cb0e0b738b078c9a7e/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01550fe5f60fd176aa66b7611289d46dc4aa4b1b904874c7b6d1d54e581c5ec1", size = 1371999, upload-time = "2026-04-07T11:14:22.63Z" }, - { url = "https://files.pythonhosted.org/packages/79/72/97a9728c711c7c1b06e107d3f0623880fb4ef90e147ed13c551a1730e7cc/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48bee0b91bebfaec41e1081e351000659ab7570cc4598d617aa04d5bf827f9e6", size = 3145715, upload-time = "2026-04-07T11:14:24.508Z" }, - { url = "https://files.pythonhosted.org/packages/ed/54/d5caabbea233ac90c286c87c260e49d7641467e87438a18d858e41c82e91/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:7e580cb04ad849ae9b786fa21383c6b994b6e6c1444ad1cb9f22392759d72741", size = 1456304, upload-time = "2026-04-07T11:14:26.515Z" }, - { url = "https://files.pythonhosted.org/packages/fc/a7/2d1a81250ac8c01a0100c026018e76f0e7a097ff63e4c553e02a6938c6fb/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:09d6c9ba091854f07817055d795d604179c12a8f308ba4c7d56f3719dfea1646", size = 2389089, upload-time = "2026-04-07T11:14:28.635Z" }, - { url = "https://files.pythonhosted.org/packages/65/0d/c47c3872203ae88e6506997c0b576ad731f5261daa25d559be09c9756658/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1e989f86113be66574113b9c7bdf4793f3f863d248e47d911b355e05ca6b6b10", size = 2493404, upload-time = "2026-04-07T11:14:30.577Z" }, - { url = "https://files.pythonhosted.org/packages/8f/2f/71e0a5a3130792146c8a200a2dd1e52aa16f7c1074012e17f2601eea9a90/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ebd1a18e2e47bc0b292a07e6ed9c3642f8aaa672d12253885f599b50807a4f9", size = 4251709, upload-time = "2026-04-07T11:14:32.451Z" }, - { url = "https://files.pythonhosted.org/packages/86/45/d39874901abacef325adb5b34ae416817c8486dfb4fb87c7a9b74ec5b072/rapidfuzz-3.14.5-cp312-cp312-win32.whl", hash = "sha256:9981d38a703b86f0e315a3cd229fd1906fe1d91c989ed121fb975b3c849f89f5", size = 1710069, upload-time = "2026-04-07T11:14:34.37Z" }, - { url = "https://files.pythonhosted.org/packages/85/0b/f65572c53de8a1c704bda707f63a447b67bdbe95d7cdc70d18885e191df5/rapidfuzz-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:d8375e3da319593389727c3187ccaf3e0e84199accc530866b8e0f2b79af05e9", size = 1540630, upload-time = "2026-04-07T11:14:36.287Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c3/143be3a578f989758cae516f3270d5cbb49783a7bfdf57cc27a670e00456/rapidfuzz-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:478b59bb018a6780d73f33e38d0b3ec5e968a6c1ed42876b993dd456b7aa20e8", size = 813137, upload-time = "2026-04-07T11:14:38.289Z" }, - { url = "https://files.pythonhosted.org/packages/11/66/252803f2010ba699618cdc048b6e1f7cc1f433c08b4a9a17579b92ab0142/rapidfuzz-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebd8fd343bf8492a1e60bcb6dc99f90f74f65d98d8241a6b3e1fed225b76ecd6", size = 1940205, upload-time = "2026-04-07T11:14:40.319Z" }, - { url = "https://files.pythonhosted.org/packages/ea/59/b2afd98e41af9cd54554a4c1c423d84cdd60e6b1c0a09496f033b55f60ec/rapidfuzz-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6737b35d5af7479c5bf9710f7b17edd9d2c43128d974d25fb4ea653e42c64609", size = 1159639, upload-time = "2026-04-07T11:14:42.52Z" }, - { url = "https://files.pythonhosted.org/packages/a3/31/7aa7e62c4c516a7af322ed0c4f0774208b72d457d0cfec808bad0df12f4a/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b002c7994cc9f2bc9d9856f0fbaee6e8072c983873846c92f25cefba5b2a925f", size = 1367194, upload-time = "2026-04-07T11:14:44.25Z" }, - { url = "https://files.pythonhosted.org/packages/90/79/2fc252a63bc91d3c3b234d0a3a6ad4ebc460037a23cdcdaf9285f986e6c9/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17a34330cd2a538c1ce5d400b61ba358c5b72c654b928ff87b362e88f8b864c7", size = 3151805, upload-time = "2026-04-07T11:14:46.21Z" }, - { url = "https://files.pythonhosted.org/packages/17/54/0c83508f2683ea70e2d05f8527eb07328acf7bb1e9d97a3bece5702378e7/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:95d937e74c1a7a1287dfb03b62a827be08ede10a155cf1af73bbf47f2b73ee6e", size = 1455667, upload-time = "2026-04-07T11:14:47.991Z" }, - { url = "https://files.pythonhosted.org/packages/71/1b/070175e873177814d58850a01ebe80e20ae11e93eb4da894d563988660fa/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:46b92a9970dcc34f0096901c792644094cab49554ac3547f35e3aebbdf0a3610", size = 2388246, upload-time = "2026-04-07T11:14:50.098Z" }, - { url = "https://files.pythonhosted.org/packages/c9/dd/77caf7aaf9c2be050ad1f128d7c24ff0f59079aa62c5f62f9df41c0af45e/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e012177c8e8a8a0754ae0d6027d63042aa5ff036d9f40f07cb3466a6082e21b8", size = 2494333, upload-time = "2026-04-07T11:14:52.303Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/dd7e1f2aa31a8fbbfc16b0610af1d770ffaf1287490f3c8c5b1c52da264f/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a2ae6f53f99c9a0eca7a0afc5b4e45fc73bc1dd4ac74c00509031d76df80ed98", size = 4258579, upload-time = "2026-04-07T11:14:54.538Z" }, - { url = "https://files.pythonhosted.org/packages/9c/0a/ac99e1ba347ba0e85e0bb60b74231d55fb93c0eff43f2920ccb413d0be08/rapidfuzz-3.14.5-cp313-cp313-win32.whl", hash = "sha256:4a60f0057231188e3bd30216f7b4e0f279b11fa4ec818bb6c1d9f014d1562fbc", size = 1709231, upload-time = "2026-04-07T11:14:56.524Z" }, - { url = "https://files.pythonhosted.org/packages/cf/cb/0e251d731b3166378644238e8f0cf9e89858c024e19f75ca9f7e3ae83fd5/rapidfuzz-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:11bfc2ed8fbe4ab86bd516fadefab126f90e6dcadffa761739fcb304707dfd35", size = 1538519, upload-time = "2026-04-07T11:14:58.635Z" }, - { url = "https://files.pythonhosted.org/packages/30/6f/4548132acc947db6d5346a248e44a8b3a22d608ef30e770fb578caaf2d00/rapidfuzz-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:b486b5218808f6f4dc471b114b1054e63553db69705c97da0271f47bd706aedd", size = 812628, upload-time = "2026-04-07T11:15:00.552Z" }, - { url = "https://files.pythonhosted.org/packages/00/60/69b177577290c5eab892c6f75fe89c3aff3f9ae80298a78d9372b1cecb9a/rapidfuzz-3.14.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:39ef8658aaf67d51667e7bdaf7096f432333377d8302ac43c70b5df8a4cf89b8", size = 1970231, upload-time = "2026-04-07T11:15:02.603Z" }, - { url = "https://files.pythonhosted.org/packages/48/38/2fd790052659cc4e2907b63c25433f0987864b445c1aeec1a302ef5ad948/rapidfuzz-3.14.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ad37a0be705b544af6296da8edddc260d10a8ae5462530fc9991f66498bb1f9", size = 1194394, upload-time = "2026-04-07T11:15:04.572Z" }, - { url = "https://files.pythonhosted.org/packages/80/f4/28430ad8472fc3536e8ebd51a864a226e979cfe924c6e3f83d111373aa74/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d45e06f60729e07d9b20c205f7e5cff90b6ef2584e852eecf46e045aea69627d", size = 1377051, upload-time = "2026-04-07T11:15:06.728Z" }, - { url = "https://files.pythonhosted.org/packages/77/7e/9aeacabcfd1e77397968362e5b98fe14248b8307011136b17daf99752a8e/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52da10236aa6212de71b9e170bace65b64b129c0dea7fc243d6c9ce976f5074", size = 3160565, upload-time = "2026-04-07T11:15:08.667Z" }, - { url = "https://files.pythonhosted.org/packages/56/f4/db4dd7be0cd2f2022117ac5407d905f435d60e48baaea313a567ad27e865/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:440d30faaf682ca496170a7f0cc5453ec942e3e079f0fd802c9a7f938dfb50a3", size = 1442113, upload-time = "2026-04-07T11:15:11.138Z" }, - { url = "https://files.pythonhosted.org/packages/a4/99/0e9f6aa57f3e32a767216f797e56dc96b720fcecfb9d8ee907ecc82f8d66/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:56227a61fd3d17b0cd9793132431f3a3d07c8654be96794ba9f89fe0fc8b2d09", size = 2396618, upload-time = "2026-04-07T11:15:13.154Z" }, - { url = "https://files.pythonhosted.org/packages/60/94/44a78e39ffce17cbdd3e2b53b696acc751d5d153be0f499d052b07a4d904/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2e83cd2e25bb4edd97b689d9979d9c3acccdaaf26ceac08212ceece202febcfa", size = 2478220, upload-time = "2026-04-07T11:15:15.193Z" }, - { url = "https://files.pythonhosted.org/packages/dd/df/454311469a09a507e9d784a35796742bec22e4cebe75551e2da4e0e290fd/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:af3b859726cd3374287e405e14b9634563c078c5531a4f62375508addebddad1", size = 4265027, upload-time = "2026-04-07T11:15:17.28Z" }, - { url = "https://files.pythonhosted.org/packages/fc/01/175465a9ab3e3b70ba669058372f009d1d49c1746e2dcd56b69df188d3a5/rapidfuzz-3.14.5-cp313-cp313t-win32.whl", hash = "sha256:8ce1d850b3c0178440efde9e884d98421b5e87ff925f364d6d79e23910d7593f", size = 1766814, upload-time = "2026-04-07T11:15:19.687Z" }, - { url = "https://files.pythonhosted.org/packages/1b/a0/a9b84a47af06ebed94a1439eb2f02adebfb8628bcd30af1fe3e02f5ef56c/rapidfuzz-3.14.5-cp313-cp313t-win_amd64.whl", hash = "sha256:c84af70bcf34e99aee894e46a0f1ac77f17d0ef828179c387407642e2466d28a", size = 1582448, upload-time = "2026-04-07T11:15:21.98Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f1/5937800238b3f8248e70860d79f69ba8f73e764fff47e36bc9e2f26dbcc6/rapidfuzz-3.14.5-cp313-cp313t-win_arm64.whl", hash = "sha256:aac0ad28c686a5e72b81668b906c030ee28050b244544b8af68e12fb32543895", size = 832932, upload-time = "2026-04-07T11:15:24.358Z" }, - { url = "https://files.pythonhosted.org/packages/81/41/aa3ffb3355e62e1bf91f6599b3092e866bc88487a07c524004943c7676df/rapidfuzz-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1a31cc6d7d03e7318a0974c038959c59e19c752b81115f2e9138b3331cd64d45", size = 1943327, upload-time = "2026-04-07T11:15:26.266Z" }, - { url = "https://files.pythonhosted.org/packages/2d/e1/c2141f1840a41e07ad2db6f724945f8f8ff3065463899a22939152dd6e09/rapidfuzz-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0298d357e2bc59d572da4db0bc631009b6f8f6c9bc8c11e99a12b833f16b6575", size = 1161755, upload-time = "2026-04-07T11:15:28.659Z" }, - { url = "https://files.pythonhosted.org/packages/ca/07/66e753eeaa353161d1d331b7dd517bb349b0bacfebe8496d7b26be26f81f/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:59b3dba758661a318995655435c6ab20a04ade79fa51e75bc8dc107cac8df280", size = 1376571, upload-time = "2026-04-07T11:15:31.225Z" }, - { url = "https://files.pythonhosted.org/packages/c8/85/9535df0b78ba51f478c9ce7eb6d1f85535cc31fe356773b48fd9d3e563ca/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4900143d82071bdda533b00300c40b14b963ff826b3642cc463b6dd0f036585e", size = 3156468, upload-time = "2026-04-07T11:15:33.428Z" }, - { url = "https://files.pythonhosted.org/packages/81/ee/b667eb93bba6dc4e0de658edd778e1619dc4d6aab68fa5e5c7f075152735/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:feedf219672eef83ea6be6f3bb093bba396a8560fc75be85ba225f082903df0a", size = 1458311, upload-time = "2026-04-07T11:15:35.557Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ce/479074f5624364a48df3403c538797ef22d3ac49c19dc76c3f79fcdcc70c/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:419e4397a36e2665ec992d8d64c20ba4b2a42500c76ecadeca78a4f19cb9cc32", size = 2398228, upload-time = "2026-04-07T11:15:37.669Z" }, - { url = "https://files.pythonhosted.org/packages/0b/15/a8982f649150fffbdcd6f17565974501f6ab33b2795267bffbd4a7ba905b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:97131ab2be39043054ee28d99e09efe316e6d53449b7e962dfcf3c2de8b2b246", size = 2497226, upload-time = "2026-04-07T11:15:39.857Z" }, - { url = "https://files.pythonhosted.org/packages/19/52/5267c03ef6759831b7d4625a0c9c06e87baa2fae084b61ac9c388858317b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:593c00dac4e30231c35bf3b4f1da8ec0998762e9e94425586a5d636fcd57f9d0", size = 4262283, upload-time = "2026-04-07T11:15:42.279Z" }, - { url = "https://files.pythonhosted.org/packages/71/c0/2579f343a97f5254c43bb5853baccc01488357dcb64a27bcb869b7888a4a/rapidfuzz-3.14.5-cp314-cp314-win32.whl", hash = "sha256:0084b687b02b4e569b46d8d6d4ad25659528e6081cd6d067ca453a69035f07e4", size = 1744614, upload-time = "2026-04-07T11:15:44.498Z" }, - { url = "https://files.pythonhosted.org/packages/17/eb/8edfed1e80119dc9c35b11df4bc701eea85622ad681fff0263b6961d3224/rapidfuzz-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:5dfa89d78f22cd773054caff44827b846161a29f2dcf7e78b8f90d086621e502", size = 1588971, upload-time = "2026-04-07T11:15:46.86Z" }, - { url = "https://files.pythonhosted.org/packages/f6/04/5676df93c85cfa57a3045d8047318df9f3cd58c7b8a99340dd95f874795e/rapidfuzz-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:67f3f9d2b444268ab53e47d31bab89954888d23c04c6789f2c727e51fe4b1d13", size = 834985, upload-time = "2026-04-07T11:15:49.411Z" }, - { url = "https://files.pythonhosted.org/packages/f7/0d/4a8988cea658fe335048ddef8c876addff1b6daa3c9ca8ad65a5a2196e69/rapidfuzz-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:77eac0526899b3c3ad1454bb2b03cdb491d67358ec8ef0c9c48bd61b632b431d", size = 1972517, upload-time = "2026-04-07T11:15:51.819Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a3/f5cfd9965a9d9a9e32249159797c47b5d6299ea6d1629f9126b25f1c10a3/rapidfuzz-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b9c6bd754d11f6e78ac54e3d86b4b11dc1ba2f13e5fc958899574532897f5a99", size = 1196056, upload-time = "2026-04-07T11:15:54.292Z" }, - { url = "https://files.pythonhosted.org/packages/64/07/561c2e40cfd10e6630a7b0ac5a2a813aef50d944bcd1f3d260319d659d5b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:738c96944d076deeaff70e92b65696ab4f7ecb8081d7791c5403a3257dfaf8ff", size = 1374732, upload-time = "2026-04-07T11:15:56.584Z" }, - { url = "https://files.pythonhosted.org/packages/c2/39/123bb94fee40e2fb3b7c49b80827c7ef42d838e18def3fc2fef5a3cf817a/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4c1bca487a17fe4226b4ffb2d30e799d2b274d692cffa76bd0746f56235fca3", size = 3166902, upload-time = "2026-04-07T11:15:58.768Z" }, - { url = "https://files.pythonhosted.org/packages/75/0a/45716fafc9fd2e028cf20b5ac5bc704887081cd312f84edb0e325599414b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:af6a90a4ed2a48fa1a2d17e9d824e6c7c950bea5bad0b707c77fd55751e6bfef", size = 1452130, upload-time = "2026-04-07T11:16:01.453Z" }, - { url = "https://files.pythonhosted.org/packages/ca/49/4e96c413114398481c0a5b0086af32c364a18613c9a2ea578d17c4bea4ee/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bf5018938208d4597b2e679a4f8cff9fd252f1df53583130ae56281a21801b64", size = 2396308, upload-time = "2026-04-07T11:16:03.588Z" }, - { url = "https://files.pythonhosted.org/packages/89/b7/49fea9fc6878d59bd259d01dd1972d9b86117992b1c66d9b16f0a65273c3/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c0919d1f89ddf91129906705723118ea09754171e4116f5a5dbc667c7bc9b261", size = 2488210, upload-time = "2026-04-07T11:16:05.871Z" }, - { url = "https://files.pythonhosted.org/packages/0c/44/a1f732b93ffacbdad077b7c801149549b2938e1bece6addb5ad85ed74df8/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93d8da883a35116d6813432177f35e570db5b0a5e30ecb0cbd7cb39c815735df", size = 4270621, upload-time = "2026-04-07T11:16:08.483Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ce/ff942d19fce5385054650bb71a58495ddda299d94661ccc4e6e7fa44868b/rapidfuzz-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:0f23e37019ec07712d58976b1ab2b889f8649a7f7c2f626a2f34ea9139e79279", size = 1803950, upload-time = "2026-04-07T11:16:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5c/0f/9aafc63f9661222b819b391c187eed29fc90ad5935f9690e5ecc2d2047a4/rapidfuzz-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7d5ca9c7832e6879a707296d1463685f7c243a27846227044504741640caec66", size = 1632357, upload-time = "2026-04-07T11:16:13.1Z" }, - { url = "https://files.pythonhosted.org/packages/70/a6/51fc1b0e61e3326e1c68a61cfd0c6b3c34c843681c4b1eefbf0596f59162/rapidfuzz-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3e91dcd2549b8f8d843f98ba03a17e01f3d8b72ce942adbbb6761bc58ffce813", size = 855409, upload-time = "2026-04-07T11:16:15.787Z" }, - { url = "https://files.pythonhosted.org/packages/d9/ee/e71853bf82846c5c2174b924b71d8e8099fb05ff87c958a720380b434ba3/rapidfuzz-3.14.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:578e6051f6d5e6200c259b47a103cf06bb875ab5814d17333fc0b5c290b22f4c", size = 1888603, upload-time = "2026-04-07T11:16:18.223Z" }, - { url = "https://files.pythonhosted.org/packages/36/82/40f67b730f32be2ebad9f62add1571c754f52249254b2e88af094b907eee/rapidfuzz-3.14.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbf1b8bb2695415b347f3727da1addca2acb82c9b97ac86bebf8b1bead1eb12d", size = 1120599, upload-time = "2026-04-07T11:16:20.682Z" }, - { url = "https://files.pythonhosted.org/packages/ef/9f/a3635cc4ec8fc6e14b46e7db1f7f8763d8c4bef33dcc124eea2e6cb2c8f3/rapidfuzz-3.14.5-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f4a8f5cc84c7ad6bffa0e9947b33eb343ad66e6b53e94fe54378a5508c5ed53", size = 1348524, upload-time = "2026-04-07T11:16:23.451Z" }, - { url = "https://files.pythonhosted.org/packages/cc/1b/2b229520f0b48464cfcd7aa758f74551d12c9bc4ab544022a60210aab064/rapidfuzz-3.14.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97c6d85283629646fa87acc22c66b30ea9d4de7f6fdf887daa2e30fa041829b5", size = 3099302, upload-time = "2026-04-07T11:16:25.858Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b5/363906b1064fc6fe611783a61764927bbd91919aaaabe8cba82151ca93ef/rapidfuzz-3.14.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:dfef96543ced67d9513a422755db422ae1dc34dade0a1485e0b43e7342ed3ebf", size = 1509889, upload-time = "2026-04-07T11:16:28.487Z" }, -] - [[package]] name = "rapidocr-onnxruntime" version = "1.4.4" @@ -3929,77 +3784,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, ] -[[package]] -name = "scipy" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" }, - { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" }, - { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" }, - { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" }, - { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" }, - { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" }, - { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" }, - { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" }, - { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" }, - { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, - { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, - { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, - { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, - { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, - { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, - { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, - { url = "https://files.pythonhosted.org/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c", size = 31590199, upload-time = "2026-02-23T00:19:17.192Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f", size = 28154001, upload-time = "2026-02-23T00:19:22.241Z" }, - { url = "https://files.pythonhosted.org/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d", size = 20325719, upload-time = "2026-02-23T00:19:26.329Z" }, - { url = "https://files.pythonhosted.org/packages/b2/83/15087d945e0e4d48ce2377498abf5ad171ae013232ae31d06f336e64c999/scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b", size = 22683595, upload-time = "2026-02-23T00:19:30.304Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, - { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, - { url = "https://files.pythonhosted.org/packages/35/e5/d6d0e51fc888f692a35134336866341c08655d92614f492c6860dc45bb2c/scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448", size = 36510943, upload-time = "2026-02-23T00:20:50.89Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fd/3be73c564e2a01e690e19cc618811540ba5354c67c8680dce3281123fb79/scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87", size = 24545621, upload-time = "2026-02-23T00:20:55.871Z" }, - { url = "https://files.pythonhosted.org/packages/6f/6b/17787db8b8114933a66f9dcc479a8272e4b4da75fe03b0c282f7b0ade8cd/scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a", size = 31936708, upload-time = "2026-02-23T00:19:58.694Z" }, - { url = "https://files.pythonhosted.org/packages/38/2e/524405c2b6392765ab1e2b722a41d5da33dc5c7b7278184a8ad29b6cb206/scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0", size = 28570135, upload-time = "2026-02-23T00:20:03.934Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c3/5bd7199f4ea8556c0c8e39f04ccb014ac37d1468e6cfa6a95c6b3562b76e/scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce", size = 20741977, upload-time = "2026-02-23T00:20:07.935Z" }, - { url = "https://files.pythonhosted.org/packages/d9/b8/8ccd9b766ad14c78386599708eb745f6b44f08400a5fd0ade7cf89b6fc93/scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6", size = 23029601, upload-time = "2026-02-23T00:20:12.161Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, - { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, - { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, - { url = "https://files.pythonhosted.org/packages/06/1c/1172a88d507a4baaf72c5a09bb6c018fe2ae0ab622e5830b703a46cc9e44/scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c", size = 36562980, upload-time = "2026-02-23T00:20:40.575Z" }, - { url = "https://files.pythonhosted.org/packages/70/b0/eb757336e5a76dfa7911f63252e3b7d1de00935d7705cf772db5b45ec238/scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49", size = 24856543, upload-time = "2026-02-23T00:20:45.313Z" }, - { url = "https://files.pythonhosted.org/packages/cf/83/333afb452af6f0fd70414dc04f898647ee1423979ce02efa75c3b0f2c28e/scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717", size = 31584510, upload-time = "2026-02-23T00:21:01.015Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a6/d05a85fd51daeb2e4ea71d102f15b34fedca8e931af02594193ae4fd25f7/scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9", size = 28170131, upload-time = "2026-02-23T00:21:05.888Z" }, - { url = "https://files.pythonhosted.org/packages/db/7b/8624a203326675d7746a254083a187398090a179335b2e4a20e2ddc46e83/scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b", size = 20342032, upload-time = "2026-02-23T00:21:09.904Z" }, - { url = "https://files.pythonhosted.org/packages/c9/35/2c342897c00775d688d8ff3987aced3426858fd89d5a0e26e020b660b301/scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866", size = 22678766, upload-time = "2026-02-23T00:21:14.313Z" }, - { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, - { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, - { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, - { url = "https://files.pythonhosted.org/packages/4b/39/f0e8ea762a764a9dc52aa7dabcfad51a354819de1f0d4652b6a1122424d6/scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19", size = 37290984, upload-time = "2026-02-23T00:22:35.023Z" }, - { url = "https://files.pythonhosted.org/packages/7c/56/fe201e3b0f93d1a8bcf75d3379affd228a63d7e2d80ab45467a74b494947/scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293", size = 25192877, upload-time = "2026-02-23T00:22:39.798Z" }, - { url = "https://files.pythonhosted.org/packages/96/ad/f8c414e121f82e02d76f310f16db9899c4fcde36710329502a6b2a3c0392/scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6", size = 31949750, upload-time = "2026-02-23T00:21:42.289Z" }, - { url = "https://files.pythonhosted.org/packages/7c/b0/c741e8865d61b67c81e255f4f0a832846c064e426636cd7de84e74d209be/scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1", size = 28585858, upload-time = "2026-02-23T00:21:47.706Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1b/3985219c6177866628fa7c2595bfd23f193ceebbe472c98a08824b9466ff/scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39", size = 20757723, upload-time = "2026-02-23T00:21:52.039Z" }, - { url = "https://files.pythonhosted.org/packages/c0/19/2a04aa25050d656d6f7b9e7b685cc83d6957fb101665bfd9369ca6534563/scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca", size = 23043098, upload-time = "2026-02-23T00:21:56.185Z" }, - { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, - { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, - { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, - { url = "https://files.pythonhosted.org/packages/11/2f/b29eafe4a3fbc3d6de9662b36e028d5f039e72d345e05c250e121a230dd4/scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484", size = 37345327, upload-time = "2026-02-23T00:22:24.442Z" }, - { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, -] - [[package]] name = "shapely" version = "2.1.2"