diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90f0aa5..eadd649 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,8 @@ jobs: - run: pip install build - run: | - rm -r src/synapt_extract/prompts 2>/dev/null || true - test -d ../../prompts && cp -r ../../prompts src/synapt_extract/prompts || true + rm -r src/synapt/extract/prompts 2>/dev/null || true + test -d ../../prompts && cp -r ../../prompts src/synapt/extract/prompts || true - run: python -m build @@ -154,7 +154,7 @@ jobs: - name: Verify Python package schemas match root schemas run: | - diff -r schemas packages/python/src/synapt_extract/schemas + diff -r schemas packages/python/src/synapt/extract/schemas echo "✅ Python package schemas match root schemas" schema-url-check: @@ -205,8 +205,8 @@ jobs: SOURCE_DATE_EPOCH: "1704067200" run: | pip install build - rm -r src/synapt_extract/prompts 2>/dev/null || true - test -d ../../prompts && cp -r ../../prompts src/synapt_extract/prompts || true + rm -r src/synapt/extract/prompts 2>/dev/null || true + test -d ../../prompts && cp -r ../../prompts src/synapt/extract/prompts || true python -m build --outdir /tmp/py-dist-1 rm -r src/synapt_extract.egg-info build 2>/dev/null || true python -m build --outdir /tmp/py-dist-2 diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 82e4fae..6660f60 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -25,8 +25,8 @@ jobs: - run: pip install build - run: | - rm -r src/synapt_extract/prompts 2>/dev/null || true - test -d ../../prompts && cp -r ../../prompts src/synapt_extract/prompts || true + rm -r src/synapt/extract/prompts 2>/dev/null || true + test -d ../../prompts && cp -r ../../prompts src/synapt/extract/prompts || true - run: python -m build diff --git a/.gitignore b/.gitignore index 6ff5241..9095198 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ !tests/security-probes/**/*.js packages/ts/prompts/ -packages/python/src/synapt_extract/prompts/ +packages/python/src/synapt/extract/prompts/ __pycache__/ *.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a50e34..4d14ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v0.6.0 + +Temporal validity role + resolution anchor — additive Stage-1 IL enrichment (config/design/extract-temporal-role-2026-07-14.md). `synapt-extract` (PyPI) bumps to 0.6.0. `@synapt-dev/extract` (npm) carries the same additive-role parity but keeps its version at 0.5.0 for now — the npm version-sync is part of the deferred full-parity effort, not this additive-role slice. + +- Added `role` (`effective` | `expiry` | `range` | `superseded` | `point`) to the temporal-ref schema, capturing the validity DIRECTION a date constrains (e.g. "expires April 30" → `expiry`, vs "effective March 2026" → `effective`) — a semantic distinction the source sentence carries but prior extraction dropped +- `role` and `resolved_end` are now BASE-tier on the `temporal_refs` capability (no longer gated behind the separate `temporal_classes` capability) — always available to any caller requesting `temporal_refs`; `type`/`context` remain `temporal_classes`-gated +- `BatchUnit` gained an optional `date` field — the unit's SOURCE date, threaded into Stage-1 as the temporal resolution anchor so partial/relative dates (e.g. "April 30") resolve against the fact's actual source year, not an unanchored guess +- The exported `SynaptTemporalRef` type declares `role` in both Python (`TypedDict`) and TypeScript (`interface`), so the public type matches the runtime emission +- Fixed a prompt-rendering gap where an absent `date` param rendered the literal string "Resolve relative dates using: None." instead of omitting the instruction (wrapped in `{{#if date}}`; supported identically by both prompt renderers) +- `_detect_capabilities`'s `temporal_classes` heuristic now keys on `type` presence only (`resolved_end` no longer implies the gated capability was exercised, since it moved to base tier) +- Published JSON schema (`schemas/temporal-ref/v1.json`, both the repo-root canonical copy and the Python package copy) updated to match — schema-drift-check green +- **ts/py parity — the additive `role` coherence slice IS included** (`@synapt-dev/extract` TypeScript): `schema.ts` types `role`, `builder.ts` emits `role`/`resolved_end` base-tier, `validate.ts` accepts/enum-validates `role` (+ `role === "range"` → `resolved_end`), `finalize.ts` drops the `resolved_end` inference, and the embedded prompt fragment matches the shared file byte-for-byte. TS and Python produce identical schema/validation/finalize output on the same inputs (verified cross-language). The FULL parity effort (a TS `extract_batch`/`batch.ts` port + version-sync) remains the post-validation follow-up per config/design/extract-ts-py-parity-plan-2026-07-15.md — this release only carries the additive-role coherence needed to keep the shared prompt/schema surface consistent. + ## v0.5.0 0.5.0 universal host-boundary groundwork. diff --git a/README.md b/README.md index eb88c00..b8a6e4d 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ console.log(result.validation); // { valid: true, errors: [] } ### Python ```python -from synapt_extract import ( +from synapt.extract import ( build_extraction_prompt, finalize_extraction, FinalizeContext, @@ -124,7 +124,7 @@ const built = builder.build({ name: "synapt_extract_stage1" }); ### Python ```python -from synapt_extract import create_extraction_builder +from synapt.extract import create_extraction_builder builder = ( create_extraction_builder(text, profile="standard") @@ -173,7 +173,7 @@ const result = await extractOpenAI(text, new OpenAI(), { ```python from openai import OpenAI -from synapt_extract import create_extraction_builder, extract_openai +from synapt.extract import create_extraction_builder, extract_openai builder = ( create_extraction_builder(text) @@ -192,7 +192,7 @@ result = await extract_openai( ) ``` -The returned result includes `artifactBundle` / `artifact_bundle`. TypeScript exports the Node artifact writer at `@synapt-dev/extract/artifacts`; Python exports `write_artifact_bundle()` from `synapt_extract`. +The returned result includes `artifactBundle` / `artifact_bundle`. TypeScript exports the Node artifact writer at `@synapt-dev/extract/artifacts`; Python exports `write_artifact_bundle()` from `synapt.extract`. ```typescript import { createExtractionBuilder, extract } from "@synapt-dev/extract"; @@ -227,7 +227,7 @@ const result = await extract(text, { ``` ```python -from synapt_extract import create_extraction_builder, extract +from synapt.extract import create_extraction_builder, extract builder = ( create_extraction_builder(text) diff --git a/SECURITY.md b/SECURITY.md index 50d123d..2ce36d6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -78,8 +78,8 @@ sha256sum *.tgz # Python (wheel) cd packages/python -cp -r ../../prompts src/synapt_extract/prompts -cp -r ../../schemas src/synapt_extract/schemas +cp -r ../../prompts src/synapt/extract/prompts +cp -r ../../schemas src/synapt/extract/schemas SOURCE_DATE_EPOCH=1704067200 python -m build sha256sum dist/*.whl ``` diff --git a/examples/dogfood.py b/examples/dogfood.py index 35c445c..f2d3b47 100644 --- a/examples/dogfood.py +++ b/examples/dogfood.py @@ -12,9 +12,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages" / "python" / "src")) import anthropic -from synapt_extract.prompt import build_extraction_prompt -from synapt_extract.finalize import FinalizeContext, finalize_extraction -from synapt_extract.validate import validate_extraction +from synapt.extract.prompt import build_extraction_prompt +from synapt.extract.finalize import FinalizeContext, finalize_extraction +from synapt.extract.validate import validate_extraction CONVERSATION = """\ Session: Weekly check-in with Marcus, April 22, 2026 diff --git a/packages/python/README.md b/packages/python/README.md index cb19dc0..d2709f3 100644 --- a/packages/python/README.md +++ b/packages/python/README.md @@ -15,7 +15,7 @@ pip install synapt-extract ## Quick start ```python -from synapt_extract import ( +from synapt.extract import ( build_extraction_prompt, finalize_extraction, FinalizeContext, @@ -50,7 +50,7 @@ assert result.validation.valid Use the builder when the model API supports structured output. It resolves capabilities once, then builds the matching prompt, Stage 1 JSON schema, OpenAI response format, finalized packet schema, and optional finalization context. ```python -from synapt_extract import create_extraction_builder +from synapt.extract import create_extraction_builder builder = ( create_extraction_builder(text, profile="standard") @@ -81,7 +81,7 @@ For OpenAI-compatible clients, use the thin adapter instead of writing callbacks ```python from openai import OpenAI -from synapt_extract import create_extraction_builder, extract_openai +from synapt.extract import create_extraction_builder, extract_openai builder = ( create_extraction_builder(text) @@ -103,7 +103,7 @@ result = await extract_openai( The returned result includes `artifact_bundle`. `write_artifact_bundle()` can also write a bundle created from any `extract()` result. ```python -from synapt_extract import create_extraction_builder, extract +from synapt.extract import create_extraction_builder, extract builder = ( create_extraction_builder(text) diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml index 68a0560..3fad947 100644 --- a/packages/python/pyproject.toml +++ b/packages/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "synapt-extract" -version = "0.5.0" +version = "0.6.0" description = "SynaptExtraction IL v1 -- schema, validation, and finalization" readme = "README.md" license = "MIT" @@ -21,17 +21,17 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries", - "Typing :: Typed", ] [project.urls] Homepage = "https://synapt.dev" Repository = "https://github.com/synapt-dev/extract" -Documentation = "https://synapt.dev/docs/extract" +Documentation = "https://synapt.dev/schemas/" Schema = "https://synapt.dev/schemas/extract/v1.json" [tool.setuptools.packages.find] where = ["src"] +namespaces = true [tool.setuptools.package-data] -synapt_extract = ["prompts/**/*.txt", "prompts/**/*.json", "schemas/**/*.json"] +"synapt.extract" = ["prompts/**/*.txt", "prompts/**/*.json", "schemas/**/*.json"] diff --git a/packages/python/src/synapt_extract/__init__.py b/packages/python/src/synapt/extract/__init__.py similarity index 80% rename from packages/python/src/synapt_extract/__init__.py rename to packages/python/src/synapt/extract/__init__.py index c52dc01..1394933 100644 --- a/packages/python/src/synapt_extract/__init__.py +++ b/packages/python/src/synapt/extract/__init__.py @@ -1,6 +1,6 @@ """synapt-extract: SynaptExtraction IL v1 schema, validation, and finalization.""" -from synapt_extract.schema import ( +from synapt.extract.schema import ( SynaptExtraction, SynaptEntity, SynaptGoal, @@ -16,9 +16,9 @@ SynaptAssertionSignals, SynaptTemporalRef, ) -from synapt_extract.validate import validate_extraction, ValidationResult, ValidationError -from synapt_extract.finalize import finalize_extraction, FinalizeContext, FinalizeResult -from synapt_extract.prompt import ( +from synapt.extract.validate import validate_extraction, ValidationResult, ValidationError +from synapt.extract.finalize import finalize_extraction, FinalizeContext, FinalizeResult +from synapt.extract.prompt import ( build_extraction_prompt, capability_embedding_input, profile_capabilities, @@ -27,14 +27,14 @@ CAPABILITY_REGISTRY, STANDARD_EMBEDDING_INPUTS, ) -from synapt_extract.builder import ( +from synapt.extract.builder import ( ExtractionBuilder, build_finalized_extraction_schema, build_extraction_schema, build_extraction_response_format, create_extraction_builder, ) -from synapt_extract.extract import ( +from synapt.extract.extract import ( extract, normalize_llm_response, run_extraction, @@ -52,15 +52,22 @@ NormalizedLlmResponse, UsageSummary, ) -from synapt_extract.artifacts import ( +from synapt.extract.artifacts import ( create_artifact_bundle, sha256_text, write_artifact_bundle, ) -from synapt_extract.openai import ( +from synapt.extract.openai import ( extract_openai, OpenAIExtractResult, ) +from synapt.extract.batch import ( + BatchFailureReason, + BatchInferRequest, + BatchUnit, + BatchUnitResult, + extract_batch, +) __all__ = [ "SynaptExtraction", @@ -116,4 +123,9 @@ "write_artifact_bundle", "extract_openai", "OpenAIExtractResult", + "BatchFailureReason", + "BatchInferRequest", + "BatchUnit", + "BatchUnitResult", + "extract_batch", ] diff --git a/packages/python/src/synapt_extract/artifacts.py b/packages/python/src/synapt/extract/artifacts.py similarity index 98% rename from packages/python/src/synapt_extract/artifacts.py rename to packages/python/src/synapt/extract/artifacts.py index dc8c3db..12e4603 100644 --- a/packages/python/src/synapt_extract/artifacts.py +++ b/packages/python/src/synapt/extract/artifacts.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from synapt_extract.extract import ExtractResult +from synapt.extract.extract import ExtractResult JsonObject = dict[str, Any] diff --git a/packages/python/src/synapt/extract/batch.py b/packages/python/src/synapt/extract/batch.py new file mode 100644 index 0000000..c42d982 --- /dev/null +++ b/packages/python/src/synapt/extract/batch.py @@ -0,0 +1,370 @@ +"""Batch Stage-1 extraction primitive for SynaptExtraction. + +Implements the pinned contract (config/design/extract-batch-limits-characterization- +2026-07-13.md §"Contract decisions") and Sentinel's spec (extract#28, +tests/python/test_extract_batch.py). Reliability logic is per-unit: shaping + +per-item validation + fail-closed fallback, with every failure contained to its +own unit slot (count-invariant). + +Why this primitive exists +------------------------- +Atlas's characterization found the generic single-text builder cannot reliably +produce a schema-valid packet even for ONE clean pre-identified unit (NO_VIABLE_N +at N=1). The failure is MALFORMATION on GROUNDED content (40/40 source-supported), +not confabulation — the model returns the right facts in the wrong shape. Fixed +with structural machinery (shaping + per-item validation + fallback), not prompt +tuning. A NEW primitive, not a wrapper/loop over the generic builder. + +Contract (pinned + spec-confirmed) +---------------------------------- + • Input: list[BatchUnit(id, text, capabilities?, date?)] — explicit attribution; the + id rides into the output as source_unit_id (boundaries stay out-of-band, never + in model-visible text). `date` (optional) is the unit's SOURCE date, threaded into + Stage-1 as the temporal resolution anchor (config/design/extract-temporal-role- + 2026-07-14.md) so partial/relative dates resolve against the source, not a guess. + • Inference: an injected `infer` seam receiving a request {prompt, messages, + capabilities} and returning a completion string. ZERO recall dependency. + • v1 strategy: PER-UNIT (one infer call per unit) — trivially out-of-band, clean + 1:1 attribution. batch-all / safe-N are future INTERNAL ladder rungs, not v1 + contract surface. Retry: one deterministic retry per failed unit (2 attempts + total), then a terminal marker from the last failure class; never replay a + successful neighbor. + • Output: COUNT-INVARIANT len(out)==len(in). Each unit → an "ok" BatchUnitResult + (valid envelope) OR a terminal {source_unit_id, status:"failed", reason} marker. + reason ∈ BatchFailureReason. No silent drops. + • Shaping (folded from recall #870/#871, held/superseded): + Class-A PRE-parse text hygiene — strip ``` fences + `//` comments, STRING- + LITERAL-AWARE (a `//` inside a JSON string, e.g. a URL, must survive). + Class-B POST-parse coercion — capability set is the arbiter: in-scope fields + coerced (scalar→array; null/non-string OPTIONAL fields like category or + decided_at are omitted; an invalid REQUIRED field is kept so strict + validation rejects it), out-of-scope dropped; temporal_refs → schema-valid + raw/resolved + base-tier role/resolved_end (type/context stay temporal_classes- + gated, so they are stripped at the base tier); non-dict leaves preserved into + strict validation. + +Harvest map: scratchpad/extract_batch_craft_harvest.md. Boundary: OSS. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Callable, Literal, TypedDict + +from synapt.extract.builder import build_extraction_schema +from synapt.extract.finalize import FinalizeContext, finalize_extraction +from synapt.extract.prompt import ( + build_extraction_prompt, + profile_capabilities, + resolve_capabilities, +) + +# Container capabilities the finalized schema always requires, even when a caller +# did not request them (mirrors recall's backfill so validation does not fail on +# containers we deliberately did not request). +_ALWAYS_BACKFILL = ("entities", "goals", "themes") +# One deterministic retry per failed unit → 2 attempts total (Q-B, Sentinel). +_MAX_ATTEMPTS = 2 + +# Terminal per-unit failure reasons (Q5). A Literal (not an Enum) so the spec's +# get_args(BatchFailureReason) reads the members. "merged" is reserved for a future +# batch-all path; the per-unit v1 path never emits it. +BatchFailureReason = Literal["unparseable", "schema_invalid", "dropped", "merged"] + + +class BatchInferRequest(TypedDict): + """The exact request the injected `infer` seam receives (Q-D). No unit id / + boundary tag ever appears here — boundaries stay in extract_batch bookkeeping, + out of model-visible text.""" + + prompt: str + messages: list[dict[str, str]] + capabilities: list[str] + + +# The injected inference seam (Q4): request → completion. The caller (recall) passes +# a model-backed callable; tests pass a deterministic/recorded one. Zero recall dep. +Inferer = Callable[[BatchInferRequest], str] + + +@dataclass +class BatchUnit: + """One pre-identified unit to extract (Q1). ``id`` is stable and rides into the + output as ``source_unit_id`` so merge/split/drop is detectable. ``capabilities`` + optionally overrides the per-call default for this unit. ``date`` is the unit's SOURCE + date (config/design/extract-temporal-role-2026-07-14.md) — the resolution anchor Stage-1 + uses to resolve partial/relative dates in ``unit.text`` (e.g. "expires April 30") against + the ACTUAL date the source material was written, not "today" or an unanchored guess. + Optional: a caller with no source date (or extracting non-temporal-sensitive units) + simply omits it, degrading gracefully to unanchored resolution.""" + + id: str + text: str + capabilities: list[str] | None = None + date: str | None = None + + +@dataclass +class BatchUnitResult: + """Per-unit outcome (Q5). ``status`` "ok" sets ``extraction``; "failed" sets + ``reason``. ``source_unit_id`` ties the slot back to its BatchUnit.""" + + source_unit_id: str + status: str # "ok" | "failed" + extraction: Any | None = None # a finalized SynaptExtraction, or None + reason: BatchFailureReason | None = None + + +async def extract_batch( + units: list[BatchUnit], + *, + infer: Inferer, + produced_by: str, + capabilities: list[str] | None = None, +) -> list[BatchUnitResult]: + """Shape + validate a batch of pre-identified units into per-unit envelopes. + + COUNT-INVARIANT: returns exactly one BatchUnitResult per input unit, in a 1:1 + slot mapping (Q5). extract_batch owns the reliability orchestration (v1 = + per-unit calls with one deterministic retry per failed unit) driven through the + injected ``infer`` seam, with zero dependency on any specific model client (Q4). + ``capabilities`` defaults to the standard profile when omitted (Q3). + """ + if not units: + return [] + ids = [unit.id for unit in units] + if len(set(ids)) != len(ids): + raise ValueError("duplicate unit id; BatchUnit ids must be unique") + + default_capabilities = ( + capabilities if capabilities is not None else profile_capabilities("standard") + ) + results: list[BatchUnitResult] = [] + for unit in units: + unit_capabilities = ( + unit.capabilities if unit.capabilities is not None else default_capabilities + ) + results.append(_extract_unit(unit, infer, produced_by, unit_capabilities)) + return results + + +def _extract_unit( + unit: BatchUnit, + infer: Inferer, + produced_by: str, + capabilities: list[str], +) -> BatchUnitResult: + """Run one unit through the reliability ladder: build an out-of-band request → + infer → Class-A hygiene + parse → Class-B coerce → finalize/validate. One + deterministic retry on failure (2 attempts total); a persisting failure yields a + terminal marker carrying the last failure's reason (Q-B).""" + reason: BatchFailureReason = "dropped" + for _attempt in range(_MAX_ATTEMPTS): + # Out-of-band: the model sees the unit TEXT only — never its id or a boundary + # tag (Q-D). The id lives in bookkeeping and rides into the packet post-hoc. + # unit.date threads as the temporal RESOLUTION anchor (config/design/extract- + # temporal-role-2026-07-14.md) — None degrades gracefully (build_extraction_prompt + # already handles an absent date). + prompt = build_extraction_prompt( + unit.text, capabilities=list(capabilities), stage="stage1", date=unit.date, + ) + request: BatchInferRequest = { + "prompt": prompt, + "messages": [{"role": "user", "content": prompt}], + "capabilities": list(capabilities), + } + # Contain the injected seam per-unit: an infer failure (e.g. RuntimeError) + # must NOT escape and void the whole batch — it is this unit's failure, + # retried once then terminal, while neighbours still produce their slots. + # No output was produced, so the closest Q5 class is "dropped". + try: + completion = infer(request) + except Exception: + reason = "dropped" + continue + parsed = _parse_completion(completion) + if parsed is None: + reason = "unparseable" + continue + + coerced = _coerce_shape(parsed, capabilities) + for key in _ALWAYS_BACKFILL: + coerced.setdefault(key, []) + context = FinalizeContext( + produced_by=produced_by, + source_id=unit.id, + capabilities_hint=list(capabilities), + ) + try: + finalized = finalize_extraction(coerced, context) + except Exception: + reason = "schema_invalid" + continue + if not finalized.validation.valid: + reason = "schema_invalid" + continue + if _is_empty_extraction(finalized.extraction, capabilities): + reason = "dropped" + continue + return BatchUnitResult( + source_unit_id=unit.id, status="ok", extraction=finalized.extraction + ) + + return BatchUnitResult(source_unit_id=unit.id, status="failed", reason=reason) + + +def _parse_completion(completion: str) -> dict | None: + """Class-A hygiene + JSON parse; None if the result is not a JSON object.""" + try: + parsed = json.loads(_strip_output_hygiene(completion)) + except (ValueError, TypeError): + return None + return parsed if isinstance(parsed, dict) else None + + +def _strip_output_hygiene(raw: str) -> str: + """Class-A PRE-parse (NET-NEW): strip ``` fences + ``//`` comments so grounded- + but-wrapped JSON parses. STRING-LITERAL-AWARE — a ``//`` inside a JSON string + value (e.g. ``https://…``) is preserved; only real line-comments are removed.""" + text = raw.strip() + # strip_markdown_fence: drop a leading ```/```json fence line, then the closing + # ``` and anything trailing it (e.g. a "Reasoning:" epilogue the model appends). + if text.startswith("```"): + newline = text.find("\n") + text = text[newline + 1:] if newline != -1 else "" + close = text.rfind("```") + if close != -1: + text = text[:close] + # strip_line_comments_outside_strings: remove `//` to end-of-line, but never when + # inside a JSON string literal (so a URL's `//` survives). Tracks string + escape. + out: list[str] = [] + in_string = False + escaped = False + i, n = 0, len(text) + while i < n: + ch = text[i] + if in_string: + out.append(ch) + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + i += 1 + elif ch == '"': + in_string = True + out.append(ch) + i += 1 + elif ch == "/" and i + 1 < n and text[i + 1] == "/": + while i < n and text[i] != "\n": + i += 1 # drop the comment body; the newline (if any) is kept next loop + else: + out.append(ch) + i += 1 + return "".join(out).strip() + + +def _coerce_shape(parsed: dict, capabilities: list[str]) -> dict: + """Class-B POST-parse (harvest ``_sanitize_stage1_output`` whitelist backbone): + the capability set is the arbiter (Q2). Per the Stage-1 schema for the requested + capabilities, whitelist each item type to its fields, coerce (scalar→array, + null/non-string optional → omit), and drop out-of-scope item types. ``entity_refs`` + is retained only when the ``entities`` capability is in scope; ``temporal_refs`` + keeps ``raw``/``resolved`` and drops schema-illegal extras (type/context/…).""" + resolved = set(resolve_capabilities(capabilities=list(capabilities))) + schema = build_extraction_schema(capabilities=list(capabilities)) + props = schema.get("properties", {}) + entities_in_scope = "entities" in resolved + + result: dict[str, Any] = {} + if "extracted_at" in parsed: + result["extracted_at"] = parsed["extracted_at"] + + for type_name, type_schema in props.items(): + if type_name == "extracted_at": + continue + if type_schema.get("type") != "array": + if type_name in parsed: + result[type_name] = parsed[type_name] + continue + items_schema = type_schema.get("items") + parsed_items = parsed.get(type_name) + if not isinstance(items_schema, dict) or "properties" not in items_schema: + result[type_name] = parsed_items if isinstance(parsed_items, list) else [] + continue + item_props = items_schema["properties"] + required = set(items_schema.get("required", [])) + coerced_items: list[Any] = [] + if isinstance(parsed_items, list): + for item in parsed_items: + if isinstance(item, dict): + coerced_items.append( + _coerce_item(item, item_props, required, entities_in_scope) + ) + else: + # Preserve non-dict leaves (null, 42, "str") verbatim so strict + # validation REJECTS them (→ schema_invalid) instead of silently + # dropping — a null sibling must fail its whole unit, not vanish. + coerced_items.append(item) + result[type_name] = coerced_items + return result + + +def _coerce_item( + item: dict, + item_props: dict, + required: set[str], + entities_in_scope: bool, +) -> dict: + """Whitelist + type-coerce one item to its schema fields. Null/non-string optional + fields are omitted (they were grounded but wrongly shaped); a scalar for an + array-typed field is wrapped; an invalid REQUIRED field is kept so finalize + rejects it (→ schema_invalid) rather than silently passing.""" + new_item: dict[str, Any] = {} + for field, field_schema in item_props.items(): + if field == "entity_refs" and not entities_in_scope: + continue # out-of-scope reference field → drop (Q2) + if field not in item: + continue + value = item[field] + field_type = field_schema.get("type") + if field_type == "array" and not isinstance(value, list): + if value is None: + continue # omit null optional array + value = [value] # scalar → array (in-scope coerce) + elif value is None: + if field in required: + new_item[field] = value # keep null required → finalize rejects + continue + elif field_type == "string" and not isinstance(value, str): + if field in required: + new_item[field] = value # keep invalid required → finalize rejects + continue + new_item[field] = value + return new_item + + +def _is_empty_extraction(extraction: Any, capabilities: list[str]) -> bool: + """True when the model produced NO payload for the unit across the REQUESTED + capabilities — every requested payload is empty. Type-aware over the Stage-1 + schema: an array payload (facts/decisions/entities/goals/…) counts when + non-empty; a scalar payload (summary/sentiment) counts when present and + non-empty. So an entities-only or summary-only extraction is NOT a false-drop. + Empty-but-valid is the 10/45 "dropped" mode: caught here and retried, never + silently absorbed.""" + schema = build_extraction_schema(capabilities=list(capabilities)) + for name, prop_schema in schema.get("properties", {}).items(): + if name == "extracted_at": + continue + value = ( + extraction.get(name) if isinstance(extraction, dict) + else getattr(extraction, name, None) + ) + if prop_schema.get("type") == "array": + if isinstance(value, list) and value: + return False + elif value not in (None, "", [], {}): + return False + return True diff --git a/packages/python/src/synapt_extract/builder.py b/packages/python/src/synapt/extract/builder.py similarity index 97% rename from packages/python/src/synapt_extract/builder.py rename to packages/python/src/synapt/extract/builder.py index 3f9d93d..14db8bf 100644 --- a/packages/python/src/synapt_extract/builder.py +++ b/packages/python/src/synapt/extract/builder.py @@ -5,8 +5,8 @@ from dataclasses import dataclass from typing import Any -from synapt_extract.finalize import FinalizeContext, FinalizeResult, finalize_extraction -from synapt_extract.prompt import ( +from synapt.extract.finalize import FinalizeContext, FinalizeResult, finalize_extraction +from synapt.extract.prompt import ( CANONICAL_ORDER, STANDARD_EMBEDDING_INPUTS, build_extraction_prompt, @@ -334,9 +334,17 @@ def _source_metadata_schema(finalized: bool = False) -> JsonSchema: def _temporal_ref_schema(capabilities: set[str], finalized: bool = False) -> JsonSchema: + # role + resolved_end are BASE-tier (config/design/extract-temporal-role-2026-07-14.md): + # always available with just the "temporal_refs" capability, NOT gated behind + # "temporal_classes" — role is the load-bearing direction signal recall's deterministic + # mapper needs, and role=="range" needs resolved_end to be usable at all. type/context stay + # temporal_classes-gated: non-load-bearing extras once role carries the direction recall + # needs; type used to be the only (weak, ambiguous) direction hint role now replaces. properties: JsonSchema = { "raw": {"type": "string"}, "resolved": {"type": "string"}, + "resolved_end": {"type": "string"}, + "role": {"type": "string", "enum": ["effective", "expiry", "range", "superseded", "point"]}, } required = ["raw"] @@ -346,7 +354,6 @@ def _temporal_ref_schema(capabilities: set[str], finalized: bool = False) -> Jso if "temporal_classes" in capabilities: properties["type"] = {"type": "string", "enum": ["point", "range", "duration", "unresolved"]} - properties["resolved_end"] = {"type": "string"} properties["context"] = {"type": "string"} required.append("type") diff --git a/packages/python/src/synapt_extract/extract.py b/packages/python/src/synapt/extract/extract.py similarity index 99% rename from packages/python/src/synapt_extract/extract.py rename to packages/python/src/synapt/extract/extract.py index 0966474..98c82ce 100644 --- a/packages/python/src/synapt_extract/extract.py +++ b/packages/python/src/synapt/extract/extract.py @@ -7,15 +7,15 @@ from dataclasses import dataclass, field from typing import Any, Awaitable, Callable, Literal, Protocol, TypeAlias, TypedDict -from synapt_extract.builder import DEFAULT_RESPONSE_FORMAT_NAME, ExtractionBuilder -from synapt_extract.finalize import FinalizeContext -from synapt_extract.prompt import ( +from synapt.extract.builder import DEFAULT_RESPONSE_FORMAT_NAME, ExtractionBuilder +from synapt.extract.finalize import FinalizeContext +from synapt.extract.prompt import ( STANDARD_EMBEDDING_INPUTS as STANDARD_EMBEDDING_INPUT_NAMES, capability_embedding_input, capability_embedding_preference, capability_name, ) -from synapt_extract.validate import ValidationResult +from synapt.extract.validate import ValidationResult JsonObject = dict[str, Any] diff --git a/packages/python/src/synapt_extract/finalize.py b/packages/python/src/synapt/extract/finalize.py similarity index 95% rename from packages/python/src/synapt_extract/finalize.py rename to packages/python/src/synapt/extract/finalize.py index c860869..c59c790 100644 --- a/packages/python/src/synapt_extract/finalize.py +++ b/packages/python/src/synapt/extract/finalize.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import Any -from synapt_extract.validate import ValidationResult, validate_extraction +from synapt.extract.validate import ValidationResult, validate_extraction @dataclass @@ -127,7 +127,11 @@ def _detect_capabilities(doc: dict[str, Any]) -> list[str]: temporal = doc.get("temporal_refs", []) if isinstance(temporal, list) and temporal: caps.append("temporal_refs") - if any(r.get("type") is not None or r.get("resolved_end") is not None for r in temporal): + # `type` alone still implies temporal_classes was exercised (it stays gated behind + # that capability). `resolved_end` no longer does — it moved to the BASE temporal_refs + # tier alongside `role` (config/design/extract-temporal-role-2026-07-14.md), so its + # presence can no longer be used to infer temporal_classes was requested. + if any(r.get("type") is not None for r in temporal): caps.append("temporal_classes") if isinstance(doc.get("language"), str): diff --git a/packages/python/src/synapt_extract/openai.py b/packages/python/src/synapt/extract/openai.py similarity index 98% rename from packages/python/src/synapt_extract/openai.py rename to packages/python/src/synapt/extract/openai.py index f788807..f5b5e40 100644 --- a/packages/python/src/synapt_extract/openai.py +++ b/packages/python/src/synapt/extract/openai.py @@ -10,8 +10,8 @@ from pathlib import Path from typing import Any -from synapt_extract.artifacts import create_artifact_bundle, write_artifact_bundle -from synapt_extract.extract import ( +from synapt.extract.artifacts import create_artifact_bundle, write_artifact_bundle +from synapt.extract.extract import ( EmbeddingRequest, EmbeddingResponse, ExtractResult, diff --git a/packages/python/src/synapt_extract/prompt.py b/packages/python/src/synapt/extract/prompt.py similarity index 98% rename from packages/python/src/synapt_extract/prompt.py rename to packages/python/src/synapt/extract/prompt.py index 81232a2..23b73c0 100644 --- a/packages/python/src/synapt_extract/prompt.py +++ b/packages/python/src/synapt/extract/prompt.py @@ -7,10 +7,12 @@ from pathlib import Path from typing import Any -from synapt_extract.schema import EXTRACTION_CAPABILITIES +from synapt.extract.schema import EXTRACTION_CAPABILITIES _INSTALLED_PROMPTS = Path(__file__).resolve().parent / "prompts" -_REPO_PROMPTS = Path(__file__).resolve().parents[4] / "prompts" +# parents[5] → repo root: this module sits at src/synapt/extract/prompt.py. +# (Pre-PEP420 it was one directory shallower and used parents[4].) +_REPO_PROMPTS = Path(__file__).resolve().parents[5] / "prompts" PROMPTS_DIR = _INSTALLED_PROMPTS if _INSTALLED_PROMPTS.is_dir() else _REPO_PROMPTS diff --git a/packages/python/src/synapt_extract/schema.py b/packages/python/src/synapt/extract/schema.py similarity index 98% rename from packages/python/src/synapt_extract/schema.py rename to packages/python/src/synapt/extract/schema.py index 7b47b91..6312025 100644 --- a/packages/python/src/synapt_extract/schema.py +++ b/packages/python/src/synapt/extract/schema.py @@ -35,6 +35,7 @@ class SynaptTemporalRef(TypedDict, total=False): version: str raw: str type: Literal["point", "range", "duration", "unresolved"] + role: Literal["effective", "expiry", "range", "superseded", "point"] resolved: str resolved_end: str context: str diff --git a/packages/python/src/synapt_extract/schemas/action/v1.json b/packages/python/src/synapt/extract/schemas/action/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/action/v1.json rename to packages/python/src/synapt/extract/schemas/action/v1.json diff --git a/packages/python/src/synapt_extract/schemas/assertion-signals/v1.json b/packages/python/src/synapt/extract/schemas/assertion-signals/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/assertion-signals/v1.json rename to packages/python/src/synapt/extract/schemas/assertion-signals/v1.json diff --git a/packages/python/src/synapt_extract/schemas/decision/v1.json b/packages/python/src/synapt/extract/schemas/decision/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/decision/v1.json rename to packages/python/src/synapt/extract/schemas/decision/v1.json diff --git a/packages/python/src/synapt_extract/schemas/embedding/v1.json b/packages/python/src/synapt/extract/schemas/embedding/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/embedding/v1.json rename to packages/python/src/synapt/extract/schemas/embedding/v1.json diff --git a/packages/python/src/synapt_extract/schemas/entity/v1.json b/packages/python/src/synapt/extract/schemas/entity/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/entity/v1.json rename to packages/python/src/synapt/extract/schemas/entity/v1.json diff --git a/packages/python/src/synapt_extract/schemas/extract/v1.json b/packages/python/src/synapt/extract/schemas/extract/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/extract/v1.json rename to packages/python/src/synapt/extract/schemas/extract/v1.json diff --git a/packages/python/src/synapt_extract/schemas/goal/v1.json b/packages/python/src/synapt/extract/schemas/goal/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/goal/v1.json rename to packages/python/src/synapt/extract/schemas/goal/v1.json diff --git a/packages/python/src/synapt_extract/schemas/producer/v1.json b/packages/python/src/synapt/extract/schemas/producer/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/producer/v1.json rename to packages/python/src/synapt/extract/schemas/producer/v1.json diff --git a/packages/python/src/synapt_extract/schemas/question/v1.json b/packages/python/src/synapt/extract/schemas/question/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/question/v1.json rename to packages/python/src/synapt/extract/schemas/question/v1.json diff --git a/packages/python/src/synapt_extract/schemas/sentiment/v1.json b/packages/python/src/synapt/extract/schemas/sentiment/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/sentiment/v1.json rename to packages/python/src/synapt/extract/schemas/sentiment/v1.json diff --git a/packages/python/src/synapt_extract/schemas/source-metadata/v1.json b/packages/python/src/synapt/extract/schemas/source-metadata/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/source-metadata/v1.json rename to packages/python/src/synapt/extract/schemas/source-metadata/v1.json diff --git a/packages/python/src/synapt_extract/schemas/source-ref/v1.json b/packages/python/src/synapt/extract/schemas/source-ref/v1.json similarity index 100% rename from packages/python/src/synapt_extract/schemas/source-ref/v1.json rename to packages/python/src/synapt/extract/schemas/source-ref/v1.json diff --git a/packages/python/src/synapt_extract/schemas/temporal-ref/v1.json b/packages/python/src/synapt/extract/schemas/temporal-ref/v1.json similarity index 76% rename from packages/python/src/synapt_extract/schemas/temporal-ref/v1.json rename to packages/python/src/synapt/extract/schemas/temporal-ref/v1.json index a82c100..cb25560 100644 --- a/packages/python/src/synapt_extract/schemas/temporal-ref/v1.json +++ b/packages/python/src/synapt/extract/schemas/temporal-ref/v1.json @@ -19,6 +19,11 @@ "enum": ["point", "range", "duration", "unresolved"], "description": "Value class. Not all temporal expressions resolve to a single timestamp." }, + "role": { + "type": "string", + "enum": ["effective", "expiry", "range", "superseded", "point"], + "description": "Validity direction this date constrains: effective (valid FROM resolved), expiry (valid UNTIL resolved), range (valid FROM resolved TO resolved_end), superseded (was valid UNTIL resolved, a prior-state fact), or point (no clear direction, fallback)." + }, "resolved": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}(T\\d{2}:\\d{2}(:\\d{2})?(\\.\\d+)?(Z|[+\\-]\\d{2}:?\\d{2})?)?$", @@ -46,6 +51,15 @@ "required": ["resolved_end"] } }, + { + "if": { + "properties": { "role": { "const": "range" } }, + "required": ["role"] + }, + "then": { + "required": ["resolved_end"] + } + }, { "if": { "properties": { "type": { "const": "unresolved" } }, diff --git a/packages/python/src/synapt_extract/validate.py b/packages/python/src/synapt/extract/validate.py similarity index 96% rename from packages/python/src/synapt_extract/validate.py rename to packages/python/src/synapt/extract/validate.py index 18832de..009e63c 100644 --- a/packages/python/src/synapt_extract/validate.py +++ b/packages/python/src/synapt/extract/validate.py @@ -6,10 +6,12 @@ from dataclasses import dataclass, field from typing import Any -from synapt_extract.schema import EXTRACTION_CAPABILITIES +from synapt.extract.schema import EXTRACTION_CAPABILITIES VALID_GOAL_STATUSES = frozenset(["open", "resolved", "abandoned", "in_progress"]) VALID_TEMPORAL_TYPES = frozenset(["point", "range", "duration", "unresolved"]) +# The validity ROLE (direction) enrichment — config/design/extract-temporal-role-2026-07-14.md. +VALID_TEMPORAL_ROLES = frozenset(["effective", "expiry", "range", "superseded", "point"]) VALID_SENTIMENT_VALENCES = frozenset(["positive", "negative", "neutral", "mixed"]) VALID_ACTION_ORIGINS = frozenset(["extracted", "proposed_from_goals"]) @@ -51,7 +53,7 @@ _RELATION_KEYS = frozenset(["target", "type", "origin", "signals"]) _SOURCE_REF_KEYS = frozenset(["version", "snippet", "offset_start", "offset_end", "sentence_index"]) _SIGNALS_KEYS = frozenset(["version", "confidence", "negated", "hedged", "condition"]) -_TEMPORAL_REF_KEYS = frozenset(["version", "raw", "type", "resolved", "resolved_end", "context"]) +_TEMPORAL_REF_KEYS = frozenset(["version", "raw", "type", "role", "resolved", "resolved_end", "context"]) _EMBEDDING_KEYS = frozenset(["version", "vector", "model", "input", "dimensions", "space", "computed_at"]) _PRODUCER_KEYS = frozenset([ "version", "model", "model_version", "deployment", "configuration", @@ -392,6 +394,19 @@ def _check_temporal_ref(obj: Any, path: str, errors: list[ValidationError]) -> N errors.append(ValidationError(f"{path}.resolved", "must not be present when type is 'unresolved'")) if "resolved_end" in obj: errors.append(ValidationError(f"{path}.resolved_end", "must not be present when type is 'unresolved'")) + # Validity ROLE (direction) — BASE-tier, optional, independent of `type` (config/design/ + # extract-temporal-role-2026-07-14.md). A separate role=="range"->resolved_end check + # mirrors the type=="range" one above, since role can appear without type now that role + # doesn't require the temporal_classes capability. + if "role" in obj: + role = obj.get("role") + if not isinstance(role, str) or role not in VALID_TEMPORAL_ROLES: + errors.append(ValidationError( + f"{path}.role", + "must be one of: effective, expiry, range, superseded, point", + )) + elif role == "range" and "resolved_end" not in obj: + errors.append(ValidationError(f"{path}.resolved_end", "required when role is 'range'")) if "resolved" in obj: if not isinstance(obj["resolved"], str) or not _is_iso_datetime(obj["resolved"]): errors.append(ValidationError(f"{path}.resolved", "must be a valid ISO 8601 date/datetime")) diff --git a/packages/ts/src/builder.ts b/packages/ts/src/builder.ts index 276f5c3..fc71bb7 100644 --- a/packages/ts/src/builder.ts +++ b/packages/ts/src/builder.ts @@ -406,9 +406,17 @@ function sourceMetadataSchema(finalized = false): JsonSchema { } function temporalRefSchema(capabilities: Set, finalized = false): JsonSchema { + // role + resolved_end are BASE-tier (config/design/extract-temporal-role-2026-07-14.md): + // always available with just the "temporal_refs" capability, NOT gated behind + // "temporal_classes" — role is the load-bearing direction signal recall's deterministic + // mapper needs, and role === "range" needs resolved_end to be usable at all. type/context + // stay temporal_classes-gated (non-load-bearing extras once role carries the direction). + // Mirrors the Python _temporal_ref_schema. const properties: JsonSchema = { raw: { type: "string" }, resolved: { type: "string" }, + resolved_end: { type: "string" }, + role: { type: "string", enum: ["effective", "expiry", "range", "superseded", "point"] }, }; const required = ["raw"]; @@ -419,7 +427,6 @@ function temporalRefSchema(capabilities: Set, finalized = if (capabilities.has("temporal_classes")) { properties.type = { type: "string", enum: ["point", "range", "duration", "unresolved"] }; - properties.resolved_end = { type: "string" }; properties.context = { type: "string" }; required.push("type"); } diff --git a/packages/ts/src/finalize.ts b/packages/ts/src/finalize.ts index 483985b..bc0c0e7 100644 --- a/packages/ts/src/finalize.ts +++ b/packages/ts/src/finalize.ts @@ -143,7 +143,10 @@ function detectCapabilities(doc: Record): ExtractionCapability[ if (Array.isArray(doc.temporal_refs) && (doc.temporal_refs as unknown[]).length > 0) { caps.push("temporal_refs"); const refs = doc.temporal_refs as Record[]; - if (refs.some((r) => r.type !== undefined || r.resolved_end !== undefined)) caps.push("temporal_classes"); + // `type` alone still implies temporal_classes was exercised (it stays gated behind that + // capability). `resolved_end` no longer does — it moved to the BASE temporal_refs tier + // alongside `role` (config/design/extract-temporal-role-2026-07-14.md). Mirrors Python. + if (refs.some((r) => r.type !== undefined)) caps.push("temporal_classes"); } if (typeof doc.language === "string") caps.push("language"); diff --git a/packages/ts/src/prompt-data.ts b/packages/ts/src/prompt-data.ts index 7d9e4d4..0afbe1e 100644 --- a/packages/ts/src/prompt-data.ts +++ b/packages/ts/src/prompt-data.ts @@ -243,6 +243,6 @@ export const EMBEDDED_PROMPT_FRAGMENTS: Record = { structured_sentiment: "- \"sentiment\": object with \"valence\" (one of \"positive\", \"negative\", \"neutral\", \"mixed\"), optional \"intensity\" (0.0-1.0), optional \"confidence\" (0.0-1.0)\n", summary: "- \"summary\": one sentence, max 200 characters\n", temporal_classes: " - include \"type\" (\"point\", \"range\", \"duration\", \"unresolved\") and \"resolved_end\" for ranges\n", - temporal_refs: "- \"temporal_refs\": array of objects with \"raw\" (as it appeared in text), \"type\" (\"point\", \"range\", \"duration\", or \"unresolved\"), \"resolved\" (ISO 8601 date/datetime for points and range starts), optional \"resolved_end\" (required for ranges), and optional \"context\". Resolve relative dates using: {{date}}. For durations, omit \"resolved\" and \"resolved_end\" unless the source gives concrete dates; never put ISO duration strings such as \"P30D\" or \"P1Y\" in \"resolved\".\n", + temporal_refs: "- \"temporal_refs\": array of objects with \"raw\" (as it appeared in text), \"role\" (\"effective\", \"expiry\", \"range\", \"superseded\", or \"point\" — the validity direction this date constrains, see below), \"type\" (\"point\", \"range\", \"duration\", or \"unresolved\"), \"resolved\" (ISO 8601 date/datetime for points and range starts), optional \"resolved_end\" (required for ranges and for role \"range\"), and optional \"context\". {{#if date}}Resolve relative dates using: {{date}}.{{/if}} For durations, omit \"resolved\" and \"resolved_end\" unless the source gives concrete dates; never put ISO duration strings such as \"P30D\" or \"P1Y\" in \"resolved\".\n- Classify each date's \"role\" — which validity boundary it constrains: \"effective\" when the text signals a start (since / from / as of / effective / started / \"migrated in March 2026\"); \"expiry\" when it signals an end (expires / until / by / deadline / ends); \"range\" when it spans from one date to another (from X to Y / between) — requires \"resolved_end\"; \"superseded\" when it describes a prior state that ended (before / \"until the refactor\" / previously); \"point\" only when there is no clear directional signal.\n", themes: "- \"themes\": array of topic strings{{#if categories}} chosen from: {{categories}}{{/if}}\n", }; diff --git a/packages/ts/src/schema.ts b/packages/ts/src/schema.ts index fc672fa..879de5f 100644 --- a/packages/ts/src/schema.ts +++ b/packages/ts/src/schema.ts @@ -28,6 +28,7 @@ export interface SynaptTemporalRef { version: "1"; raw: string; type?: "point" | "range" | "duration" | "unresolved"; + role?: "effective" | "expiry" | "range" | "superseded" | "point"; resolved?: string; resolved_end?: string; context?: string; diff --git a/packages/ts/src/validate.ts b/packages/ts/src/validate.ts index 279cedd..5f38461 100644 --- a/packages/ts/src/validate.ts +++ b/packages/ts/src/validate.ts @@ -20,6 +20,11 @@ const VALID_TEMPORAL_TYPES: Set = new Set([ "point", "range", "duration", "unresolved", ]); +// The validity ROLE (direction) enrichment — config/design/extract-temporal-role-2026-07-14.md. +const VALID_TEMPORAL_ROLES: Set = new Set([ + "effective", "expiry", "range", "superseded", "point", +]); + const VALID_SENTIMENT_VALENCES: Set = new Set([ "positive", "negative", "neutral", "mixed", ]); @@ -58,7 +63,7 @@ const SOURCE_METADATA_KEYS = new Set(["version", "token_count", "character_count const RELATION_KEYS = new Set(["target", "type", "origin", "signals"]); const SOURCE_REF_KEYS = new Set(["version", "snippet", "offset_start", "offset_end", "sentence_index"]); const SIGNALS_KEYS = new Set(["version", "confidence", "negated", "hedged", "condition"]); -const TEMPORAL_REF_KEYS = new Set(["version", "raw", "type", "resolved", "resolved_end", "context"]); +const TEMPORAL_REF_KEYS = new Set(["version", "raw", "type", "role", "resolved", "resolved_end", "context"]); const EMBEDDING_KEYS = new Set(["version", "vector", "model", "input", "dimensions", "space", "computed_at"]); const PRODUCER_KEYS = new Set([ "version", "model", "model_version", "deployment", "configuration", @@ -533,6 +538,17 @@ function validateTemporalRef(obj: unknown, path: string, errors: ValidationError } } } + // Validity ROLE (direction) — BASE-tier, optional, independent of `type` (config/design/ + // extract-temporal-role-2026-07-14.md). A separate role === "range" -> resolved_end check + // mirrors the type === "range" one above, since role can appear without type now that role + // doesn't require the temporal_classes capability. Mirrors the Python _check_temporal_ref. + if (ref.role !== undefined) { + if (typeof ref.role !== "string" || !VALID_TEMPORAL_ROLES.has(ref.role)) { + errors.push({ path: `${path}.role`, message: "must be one of: effective, expiry, range, superseded, point" }); + } else if (ref.role === "range" && ref.resolved_end === undefined) { + errors.push({ path: `${path}.resolved_end`, message: "required when role is 'range'" }); + } + } if (ref.resolved !== undefined) { if (typeof ref.resolved !== "string" || !isIsoDatetime(ref.resolved)) { errors.push({ path: `${path}.resolved`, message: "must be a valid ISO 8601 date/datetime" }); diff --git a/packages/ts/tests/test_finalize.ts b/packages/ts/tests/test_finalize.ts index 2f2d289..b652b93 100644 --- a/packages/ts/tests/test_finalize.ts +++ b/packages/ts/tests/test_finalize.ts @@ -193,6 +193,21 @@ describe("finalizeExtraction", () => { expect(result.extraction.capabilities).toContain(capability); }); + test("resolved_end alone does not imply temporal_classes (base-tier role)", () => { + // role + resolved_end are BASE-tier (config/design/extract-temporal-role-2026-07-14.md) — + // a range-role ref can carry resolved_end WITHOUT temporal_classes ever being exercised. + // The old heuristic (type OR resolved_end) would mislabel this; only `type` should trigger + // detection now. Mirrors the Python test_resolved_end_alone_does_not_imply_temporal_classes. + const result = finalizeExtraction( + llmOutput({ + temporal_refs: [{ raw: "March to April 2026", role: "range", resolved: "2026-03-01", resolved_end: "2026-04-30" }], + }), + { produced_by: "test://model" }, + ); + expect(result.extraction.capabilities).toContain("temporal_refs"); + expect(result.extraction.capabilities).not.toContain("temporal_classes"); + }); + test("warns on mismatched capabilities hint", () => { const result = finalizeExtraction(llmOutput(), { produced_by: "test://model", diff --git a/packages/ts/tests/test_prompt.ts b/packages/ts/tests/test_prompt.ts index 4d9d7ae..203ca5b 100644 --- a/packages/ts/tests/test_prompt.ts +++ b/packages/ts/tests/test_prompt.ts @@ -161,6 +161,25 @@ describe("buildExtractionPrompt", () => { expect(result).toContain("2026-04-25"); }); + test("temporal_refs fragment carries role classification instructions", () => { + // config/design/extract-temporal-role-2026-07-14.md — the Stage-1 prompt classifies each + // temporal ref's validity role, with the 5 enum values named. Mirrors the Python test. + const result = buildExtractionPrompt(SAMPLE_TEXT, { capabilities: ["temporal_refs"] }); + expect(result).toContain('"role"'); + for (const role of ["effective", "expiry", "range", "superseded", "point"]) { + expect(result).toContain(role); + } + }); + + test("temporal_refs omits the resolve instruction (no literal 'None') when no date given", () => { + // Regression guard mirroring the Python test: without a date the {{#if date}} conditional + // must render nothing, NOT the literal "Resolve relative dates using: None." The TS + // renderer supports {{#if}} identically to Python (prompt.ts), so this holds cross-language. + const result = buildExtractionPrompt(SAMPLE_TEXT, { capabilities: ["temporal_refs"] }); + expect(result).not.toContain("using: None"); + expect(result).not.toContain("None."); + }); + test.each([ ["minimal", false, false], ["standard", false, true], @@ -481,6 +500,30 @@ describe("buildExtractionSchema", () => { expect(temporalRef.required).toEqual(["version", "raw"]); }); + test("temporal role + resolved_end are base-tier; type/context stay temporal_classes-gated", () => { + // config/design/extract-temporal-role-2026-07-14.md — role is the load-bearing direction + // signal, always available with just "temporal_refs"; type/context remain gated. Mirrors + // the Python base-tier coverage (test_role_and_resolved_end_survive_coercion + the + // type/context negative control). Requesting ONLY temporal_refs, NOT temporal_classes. + const base = buildExtractionSchema({ capabilities: ["temporal_refs"] }); + const baseProps = ((base.properties as Record).temporal_refs as { + items: { properties: Record }; + }).items.properties; + expect(baseProps.role).toBeDefined(); + expect(baseProps.resolved_end).toBeDefined(); + expect(baseProps.type).toBeUndefined(); // temporal_classes-gated + expect(baseProps.context).toBeUndefined(); // temporal_classes-gated + expect(baseProps.role).toEqual({ type: "string", enum: ["effective", "expiry", "range", "superseded", "point"] }); + + const gated = buildExtractionSchema({ capabilities: ["temporal_refs", "temporal_classes"] }); + const gatedProps = ((gated.properties as Record).temporal_refs as { + items: { properties: Record }; + }).items.properties; + expect(gatedProps.type).toBeDefined(); // now present with temporal_classes + expect(gatedProps.context).toBeDefined(); + expect(gatedProps.role).toBeDefined(); // role still present (base-tier, unaffected) + }); + test("schema covers all v1.2 capability fields", () => { const schema = buildExtractionSchema({ profile: "full" }); const properties = schema.properties as Record; diff --git a/packages/ts/tests/test_validate.ts b/packages/ts/tests/test_validate.ts index 98a044d..282d890 100644 --- a/packages/ts/tests/test_validate.ts +++ b/packages/ts/tests/test_validate.ts @@ -310,6 +310,58 @@ describe("validateExtraction", () => { false, ["temporal_refs[0].resolved_end"], ], + // Validity ROLE (direction) enrichment — config/design/extract-temporal-role-2026-07-14.md. + // Mirrors the Python TestTemporalRefRole cases. role + resolved_end are BASE-tier (no + // temporal_classes capability needed); role is enum-constrained; role === "range" needs + // resolved_end. + [ + "valid role effective (base-tier, no type)", + minimalExtraction({ + temporal_refs: [{ version: "1", raw: "effective March 2026", role: "effective", resolved: "2026-03-01" }], + }), + true, + [], + ], + [ + "valid role expiry", + minimalExtraction({ + temporal_refs: [{ version: "1", raw: "expires April 30", role: "expiry", resolved: "2026-04-30" }], + }), + true, + [], + ], + [ + "valid role range with resolved_end", + minimalExtraction({ + temporal_refs: [{ version: "1", raw: "March to April", role: "range", resolved: "2026-03-01", resolved_end: "2026-04-30" }], + }), + true, + [], + ], + [ + "role absent still valid (backward compat)", + minimalExtraction({ + temporal_refs: [{ version: "1", raw: "next Tuesday", resolved: "2026-04-28" }], + }), + true, + [], + ], + [ + "invalid role value rejected", + minimalExtraction({ + temporal_refs: [{ version: "1", raw: "sometime", role: "urgent", resolved: "2026-04-28" }], + }), + false, + ["temporal_refs[0].role"], + ], + [ + "role range without resolved_end rejected", + minimalExtraction({ + temporal_refs: [{ version: "1", raw: "April 20 to May 1", role: "range", resolved: "2026-04-20" }], + }), + false, + ["temporal_refs[0].resolved_end"], + ], ])("%s", (_name, doc, expectedValid, paths) => { const result = validateExtraction(doc); expect(result.valid).toBe(expectedValid); diff --git a/prompts/v1/temporal_refs.txt b/prompts/v1/temporal_refs.txt index 8144a79..b4a0e1c 100644 --- a/prompts/v1/temporal_refs.txt +++ b/prompts/v1/temporal_refs.txt @@ -1 +1,2 @@ -- "temporal_refs": array of objects with "raw" (as it appeared in text), "type" ("point", "range", "duration", or "unresolved"), "resolved" (ISO 8601 date/datetime for points and range starts), optional "resolved_end" (required for ranges), and optional "context". Resolve relative dates using: {{date}}. For durations, omit "resolved" and "resolved_end" unless the source gives concrete dates; never put ISO duration strings such as "P30D" or "P1Y" in "resolved". +- "temporal_refs": array of objects with "raw" (as it appeared in text), "role" ("effective", "expiry", "range", "superseded", or "point" — the validity direction this date constrains, see below), "type" ("point", "range", "duration", or "unresolved"), "resolved" (ISO 8601 date/datetime for points and range starts), optional "resolved_end" (required for ranges and for role "range"), and optional "context". {{#if date}}Resolve relative dates using: {{date}}.{{/if}} For durations, omit "resolved" and "resolved_end" unless the source gives concrete dates; never put ISO duration strings such as "P30D" or "P1Y" in "resolved". +- Classify each date's "role" — which validity boundary it constrains: "effective" when the text signals a start (since / from / as of / effective / started / "migrated in March 2026"); "expiry" when it signals an end (expires / until / by / deadline / ends); "range" when it spans from one date to another (from X to Y / between) — requires "resolved_end"; "superseded" when it describes a prior state that ended (before / "until the refactor" / previously); "point" only when there is no clear directional signal. diff --git a/schemas/temporal-ref/v1.json b/schemas/temporal-ref/v1.json index a82c100..cb25560 100644 --- a/schemas/temporal-ref/v1.json +++ b/schemas/temporal-ref/v1.json @@ -19,6 +19,11 @@ "enum": ["point", "range", "duration", "unresolved"], "description": "Value class. Not all temporal expressions resolve to a single timestamp." }, + "role": { + "type": "string", + "enum": ["effective", "expiry", "range", "superseded", "point"], + "description": "Validity direction this date constrains: effective (valid FROM resolved), expiry (valid UNTIL resolved), range (valid FROM resolved TO resolved_end), superseded (was valid UNTIL resolved, a prior-state fact), or point (no clear direction, fallback)." + }, "resolved": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}(T\\d{2}:\\d{2}(:\\d{2})?(\\.\\d+)?(Z|[+\\-]\\d{2}:?\\d{2})?)?$", @@ -46,6 +51,15 @@ "required": ["resolved_end"] } }, + { + "if": { + "properties": { "role": { "const": "range" } }, + "required": ["role"] + }, + "then": { + "required": ["resolved_end"] + } + }, { "if": { "properties": { "type": { "const": "unresolved" } }, diff --git a/tests/python/fixtures/extract-batch-real-failures-v1.json b/tests/python/fixtures/extract-batch-real-failures-v1.json new file mode 100644 index 0000000..aa4e8ae --- /dev/null +++ b/tests/python/fixtures/extract-batch-real-failures-v1.json @@ -0,0 +1,2951 @@ +{ + "fixture_set": "extract-batch-real-failures-2026-07-13", + "fixture_schema_version": 1, + "source": { + "repository": "synapt-dev/config", + "merge_commit": "3b3d528", + "leaf_labels_path": "config/design/results/extract-batch-limits-2026-07-13/manual-leaf-labels.jsonl", + "leaf_labels_sha256": "272599669916bb2eec16aaebeca6ccee50dde3ef05bc6ae0fd1d51436e70ca6f", + "unit_labels_path": "config/design/results/extract-batch-limits-2026-07-13/manual-unit-labels.jsonl", + "unit_labels_sha256": "362c8412fb862ffc6cb300ae2b1b68218e58b7bc4d2a4313b78bb2a84e298764", + "raw_results_path": "config/design/results/extract-batch-limits-2026-07-13/raw-results.jsonl", + "raw_results_sha256": "30412251126a4f1fe83ae3c2329c3c65416db21a9779a246feeb5b6ab14e68da" + }, + "boundary": { + "target": "OSS synapt-extract tests", + "review": "The selected sensitivity content contains public product/process facts only. No unpublished scores, private implementation, secrets, or user data are included." + }, + "interpretation": { + "normalization_scope": "Repair envelope and leaf shape only. Preserve text and semantic placement; do not silently rewrite wrong-category or incomplete content.", + "per_unit_scope": "Every structured input unit receives one attributable terminal result: a schema-valid envelope or an explicit fail-closed marker.", + "forbidden": [ + "in-band unit delimiters", + "silent source-unit drops", + "counting analysis-only repair as an empirical success", + "reporting contract-derived scalar coercion as observed data" + ] + }, + "counts": { + "exact_raw_response_cases": 21, + "exact_malformed_sensitivity_leaves": 25, + "exact_dropped_source_occurrences": 10, + "exact_unknown_key_leaves": 2, + "exact_temporal_shape_cases": 3, + "contract_derived_cases": 1, + "raw_responses_with_markdown_fences": 21, + "raw_responses_with_line_comments": 15, + "malformed_sensitivity_error_counts": { + "category_not_string": 15, + "decided_at_not_string": 8, + "entity_refs_not_string_array": 10 + } + }, + "raw_response_cases": [ + { + "fixture_id": "raw-response::sensitivity-bullets_mixed_N02-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_mixed_N02-r01", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_mixed_N02", + "seed": 202607150500, + "input_units": [ + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\",\n \"facts\": [\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n }\n ],\n \"decisions\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"entity_refs\": null,\n \"decided_at\": null\n }\n ],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "decisions[0]:omit_null_or_invalid_entity_refs", + "decisions[0]:omit_null_or_non_string_decided_at" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "The premium package is proprietary." + } + ], + "decisions": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level" + } + ], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N02-r01::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N02-r01::decisions::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 2 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_mixed_N02-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_mixed_N02-r02", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_mixed_N02", + "seed": 202607150501, + "input_units": [ + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO 8601 timestamp\n \"facts\": [\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n }\n ],\n \"decisions\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"entity_refs\": null,\n \"decided_at\": null\n },\n {\n \"text\": \"because one extract call produces one envelope rather than one packet per input unit\",\n \"entity_refs\": null,\n \"decided_at\": null\n }\n ],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "decisions[0]:omit_null_or_invalid_entity_refs", + "decisions[0]:omit_null_or_non_string_decided_at", + "decisions[1]:omit_null_or_invalid_entity_refs", + "decisions[1]:omit_null_or_non_string_decided_at" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "The premium package is proprietary." + } + ], + "decisions": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level" + }, + { + "text": "because one extract call produces one envelope rather than one packet per input unit" + } + ], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N02-r02::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N02-r02::decisions::0", + "sensitivity-bullets_mixed_N02-r02::decisions::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 2 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_mixed_N02-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_mixed_N02-r03", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_mixed_N02", + "seed": 202607150502, + "input_units": [ + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\",\n \"facts\": [\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n }\n ],\n \"decisions\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"entity_refs\": null,\n \"decided_at\": null\n },\n {\n \"text\": \"because one extract call produces one envelope rather than one packet per input unit\",\n \"entity_refs\": null,\n \"decided_at\": null\n }\n ],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "decisions[0]:omit_null_or_invalid_entity_refs", + "decisions[0]:omit_null_or_non_string_decided_at", + "decisions[1]:omit_null_or_invalid_entity_refs", + "decisions[1]:omit_null_or_non_string_decided_at" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "The premium package is proprietary." + } + ], + "decisions": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level" + }, + { + "text": "because one extract call produces one envelope rather than one packet per input unit" + } + ], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N02-r03::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N02-r03::decisions::0", + "sensitivity-bullets_mixed_N02-r03::decisions::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 2 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_mixed_N04-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_mixed_N04-r01", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_mixed_N04", + "seed": 202607150600, + "input_units": [ + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + }, + { + "unit_id": "M09", + "pool": "multi", + "text": "On June 22, 2026, the team established that independent Modal variants should run concurrently with spawn.", + "expected_leaf_count": 1, + "expected_type": "decision" + }, + { + "unit_id": "M01", + "pool": "multi", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\",\n \"facts\": [\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n }\n ],\n \"decisions\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.\",\n \"decided_at\": null,\n \"entity_refs\": null\n },\n {\n \"text\": \"independent Modal variants should run concurrently with spawn.\",\n \"decided_at\": \"2026-06-22T00:00:00Z\",\n \"entity_refs\": null\n }\n ],\n \"temporal_refs\": [\n {\n \"raw\": \"June 22, 2026\",\n \"type\": \"point\",\n \"resolved\": \"2026-06-22T00:00:00Z\",\n \"context\": \"decision reference\"\n }\n ]\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "decisions[0]:omit_null_or_invalid_entity_refs", + "decisions[0]:omit_null_or_non_string_decided_at", + "decisions[1]:omit_null_or_invalid_entity_refs", + "temporal_refs[0]:drop_unknown_key:context", + "temporal_refs[0]:drop_unknown_key:type" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "The premium package is proprietary." + } + ], + "decisions": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit." + }, + { + "text": "independent Modal variants should run concurrently with spawn.", + "decided_at": "2026-06-22T00:00:00Z" + } + ], + "temporal_refs": [ + { + "raw": "June 22, 2026", + "resolved": "2026-06-22T00:00:00Z" + } + ] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r01::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r01::decisions::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M09", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r01::decisions::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "M01" + ], + "required_terminal_result_count": 4 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_mixed_N04-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_mixed_N04-r02", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_mixed_N04", + "seed": 202607150601, + "input_units": [ + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + }, + { + "unit_id": "M09", + "pool": "multi", + "text": "On June 22, 2026, the team established that independent Modal variants should run concurrently with spawn.", + "expected_leaf_count": 1, + "expected_type": "decision" + }, + { + "unit_id": "M01", + "pool": "multi", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\",\n \"facts\": [\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n }\n ],\n \"decisions\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.\",\n \"decided_at\": null,\n \"entity_refs\": null\n },\n {\n \"text\": \"independent Modal variants should run concurrently with spawn.\",\n \"decided_at\": \"2026-06-22T00:00:00Z\",\n \"entity_refs\": null\n }\n ],\n \"temporal_refs\": [\n {\n \"raw\": \"June 22, 2026\",\n \"type\": \"point\",\n \"resolved\": \"2026-06-22T00:00:00Z\",\n \"context\": \"decision reference\"\n }\n ]\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "decisions[0]:omit_null_or_invalid_entity_refs", + "decisions[0]:omit_null_or_non_string_decided_at", + "decisions[1]:omit_null_or_invalid_entity_refs", + "temporal_refs[0]:drop_unknown_key:context", + "temporal_refs[0]:drop_unknown_key:type" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "The premium package is proprietary." + } + ], + "decisions": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit." + }, + { + "text": "independent Modal variants should run concurrently with spawn.", + "decided_at": "2026-06-22T00:00:00Z" + } + ], + "temporal_refs": [ + { + "raw": "June 22, 2026", + "resolved": "2026-06-22T00:00:00Z" + } + ] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r02::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r02::decisions::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M09", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r02::decisions::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "M01" + ], + "required_terminal_result_count": 4 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_mixed_N04-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_mixed_N04-r03", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_mixed_N04", + "seed": 202607150602, + "input_units": [ + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + }, + { + "unit_id": "M09", + "pool": "multi", + "text": "On June 22, 2026, the team established that independent Modal variants should run concurrently with spawn.", + "expected_leaf_count": 1, + "expected_type": "decision" + }, + { + "unit_id": "M01", + "pool": "multi", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\",\n \"facts\": [\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n },\n {\n \"text\": \"The team established that independent Modal variants should run concurrently with spawn on June 22, 2026.\",\n \"category\": \"temporal_decision\"\n }\n ],\n \"decisions\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"entity_refs\": null,\n \"decided_at\": null\n }\n ],\n \"temporal_refs\": [\n {\n \"raw\": \"June 22, 2026\",\n \"type\": \"point\",\n \"resolved\": \"2026-06-22\",\n \"context\": \"Establishment of concurrent spawn for Modal variants\"\n }\n ]\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "decisions[0]:omit_null_or_invalid_entity_refs", + "decisions[0]:omit_null_or_non_string_decided_at", + "temporal_refs[0]:drop_unknown_key:context", + "temporal_refs[0]:drop_unknown_key:type" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "The premium package is proprietary." + }, + { + "text": "The team established that independent Modal variants should run concurrently with spawn on June 22, 2026.", + "category": "temporal_decision" + } + ], + "decisions": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level" + } + ], + "temporal_refs": [ + { + "raw": "June 22, 2026", + "resolved": "2026-06-22" + } + ] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r03::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r03::decisions::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M09", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_mixed_N04-r03::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "M01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "M01" + ], + "required_terminal_result_count": 4 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_simple_N02-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_simple_N02-r01", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_simple_N02", + "seed": 202607150300, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S02", + "pool": "simple", + "text": "The recall package is open source.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO timestamp\n \"facts\": [\n {\n \"text\": \"Synapt uses gitgrip to coordinate multiple repositories.\",\n \"category\": null\n },\n {\n \"text\": \"The recall package is open source.\",\n \"category\": null\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "facts[1]:omit_null_or_non_string_category" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + { + "text": "The recall package is open source." + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N02-r01::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S02", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N02-r01::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 2 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_simple_N02-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_simple_N02-r02", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_simple_N02", + "seed": 202607150301, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S02", + "pool": "simple", + "text": "The recall package is open source.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current timestamp\n \"facts\": [\n {\n \"text\": \"Synapt uses gitgrip to coordinate multiple repositories.\",\n \"category\": null\n },\n {\n \"text\": \"The recall package is open source.\",\n \"category\": null\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "facts[1]:omit_null_or_non_string_category" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + { + "text": "The recall package is open source." + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N02-r02::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S02", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N02-r02::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 2 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_simple_N02-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_simple_N02-r03", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_simple_N02", + "seed": 202607150302, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S02", + "pool": "simple", + "text": "The recall package is open source.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO 8601 timestamp\n \"facts\": [\n {\n \"text\": \"Synapt uses gitgrip to coordinate multiple repositories.\",\n \"category\": null\n },\n {\n \"text\": \"The recall package is open source.\",\n \"category\": null\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "facts[1]:omit_null_or_non_string_category" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + { + "text": "The recall package is open source." + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N02-r03::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S02", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N02-r03::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 2 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_simple_N04-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_simple_N04-r01", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_simple_N04", + "seed": 202607150400, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S02", + "pool": "simple", + "text": "The recall package is open source.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S04", + "pool": "simple", + "text": "The extract library emits SynaptExtraction documents.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO 8601 timestamp\n \"facts\": [\n {\n \"text\": \"Synapt uses gitgrip to coordinate multiple repositories.\",\n \"category\": null\n },\n {\n \"text\": \"The recall package is open source.\",\n \"category\": null\n },\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": null\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [ + "facts[0]:omit_null_or_non_string_category", + "facts[1]:omit_null_or_non_string_category", + "facts[2]:omit_null_or_non_string_category" + ], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + { + "text": "The recall package is open source." + }, + { + "text": "The premium package is proprietary." + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r01::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S02", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r01::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r01::facts::2" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S04", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "S04" + ], + "required_terminal_result_count": 4 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_simple_N04-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_simple_N04-r02", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_simple_N04", + "seed": 202607150401, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S02", + "pool": "simple", + "text": "The recall package is open source.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S04", + "pool": "simple", + "text": "The extract library emits SynaptExtraction documents.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for actual extraction time\n \"facts\": [\n {\n \"text\": \"Synapt uses gitgrip to coordinate multiple repositories.\",\n \"category\": \"coordination\"\n },\n {\n \"text\": \"The recall package is open source.\",\n \"category\": \"licensing\"\n },\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": \"licensing\"\n },\n {\n \"text\": \"The extract library emits SynaptExtraction documents.\",\n \"category\": \"output_format\"\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "category": "coordination" + }, + { + "text": "The recall package is open source.", + "category": "licensing" + }, + { + "text": "The premium package is proprietary.", + "category": "licensing" + }, + { + "text": "The extract library emits SynaptExtraction documents.", + "category": "output_format" + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r02::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S02", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r02::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r02::facts::2" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S04", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r02::facts::3" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 4 + }, + { + "fixture_id": "raw-response::sensitivity-bullets_simple_N04-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-bullets_simple_N04-r03", + "serialization": "numbered_bullets", + "sensitivity_cell": "bullets_simple_N04", + "seed": 202607150402, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S02", + "pool": "simple", + "text": "The recall package is open source.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S03", + "pool": "simple", + "text": "The premium package is proprietary.", + "expected_leaf_count": 1, + "expected_type": "fact" + }, + { + "unit_id": "S04", + "pool": "simple", + "text": "The extract library emits SynaptExtraction documents.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for actual extraction time\n \"facts\": [\n {\n \"text\": \"Synapt uses gitgrip to coordinate multiple repositories.\",\n \"category\": \"coordination\"\n },\n {\n \"text\": \"The recall package is open source.\",\n \"category\": \"licensing\"\n },\n {\n \"text\": \"The premium package is proprietary.\",\n \"category\": \"licensing\"\n },\n {\n \"text\": \"The extract library emits SynaptExtraction documents.\",\n \"category\": \"output_format\"\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "category": "coordination" + }, + { + "text": "The recall package is open source.", + "category": "licensing" + }, + { + "text": "The premium package is proprietary.", + "category": "licensing" + }, + { + "text": "The extract library emits SynaptExtraction documents.", + "category": "output_format" + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r03::facts::0" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S02", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r03::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S03", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r03::facts::2" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + }, + { + "source_unit_id": "S04", + "observed_mapped_leaf_ids": [ + "sensitivity-bullets_simple_N04-r03::facts::3" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 4 + }, + { + "fixture_id": "raw-response::sensitivity-raw_decision-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_decision-r01", + "serialization": "raw", + "sensitivity_cell": "raw_decision", + "seed": 202607150200, + "input_units": [ + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for actual extraction time\n \"facts\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"category\": \"technical\"\n },\n {\n \"text\": \"one extract call produces one envelope rather than one packet per input unit\",\n \"category\": \"implementation_detail\"\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level", + "category": "technical" + }, + { + "text": "one extract call produces one envelope rather than one packet per input unit", + "category": "implementation_detail" + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-raw_decision-r01::facts::0", + "sensitivity-raw_decision-r01::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_decision-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_decision-r02", + "serialization": "raw", + "sensitivity_cell": "raw_decision", + "seed": 202607150201, + "input_units": [ + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for actual extraction timestamp\n \"facts\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"category\": \"technical\"\n },\n {\n \"text\": \"one extract call produces one envelope rather than one packet per input unit\",\n \"category\": \"implementation\"\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level", + "category": "technical" + }, + { + "text": "one extract call produces one envelope rather than one packet per input unit", + "category": "implementation" + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-raw_decision-r02::facts::0", + "sensitivity-raw_decision-r02::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_decision-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_decision-r03", + "serialization": "raw", + "sensitivity_cell": "raw_decision", + "seed": 202607150202, + "input_units": [ + { + "unit_id": "M05", + "pool": "multi", + "text": "The team decided to measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "expected_leaf_count": 1, + "expected_type": "decision" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for actual extraction timestamp\n \"facts\": [\n {\n \"text\": \"measure fidelity at the facts-and-decisions leaf level\",\n \"category\": \"technical\"\n },\n {\n \"text\": \"one extract call produces one envelope rather than one packet per input unit\",\n \"category\": \"implementation_detail\"\n }\n ],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [ + { + "text": "measure fidelity at the facts-and-decisions leaf level", + "category": "technical" + }, + { + "text": "one extract call produces one envelope rather than one packet per input unit", + "category": "implementation_detail" + } + ], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "M05", + "observed_mapped_leaf_ids": [ + "sensitivity-raw_decision-r03::facts::0", + "sensitivity-raw_decision-r03::facts::1" + ], + "observed_primary_status": "present", + "required_terminal_slot": true, + "required_fallback": false, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_multi_clause-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_multi_clause-r01", + "serialization": "raw", + "sensitivity_cell": "raw_multi_clause", + "seed": 202607150100, + "input_units": [ + { + "unit_id": "M01", + "pool": "multi", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO timestamp\n \"facts\": [],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "M01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "M01" + ], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_multi_clause-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_multi_clause-r02", + "serialization": "raw", + "sensitivity_cell": "raw_multi_clause", + "seed": 202607150101, + "input_units": [ + { + "unit_id": "M01", + "pool": "multi", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO 8601 timestamp\n \"facts\": [],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "M01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "M01" + ], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_multi_clause-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_multi_clause-r03", + "serialization": "raw", + "sensitivity_cell": "raw_multi_clause", + "seed": 202607150102, + "input_units": [ + { + "unit_id": "M01", + "pool": "multi", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Hypothetical timestamp (replace with actual extraction time)\n \"facts\": [],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "M01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "M01" + ], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_simple_fact-r01", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_simple_fact-r01", + "serialization": "raw", + "sensitivity_cell": "raw_simple_fact", + "seed": 202607150000, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2024-02-20T00:00:00Z\",\n \"facts\": [],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2024-02-20T00:00:00Z", + "facts": [], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "S01" + ], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_simple_fact-r02", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_simple_fact-r02", + "serialization": "raw", + "sensitivity_cell": "raw_simple_fact", + "seed": 202607150001, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Auto-assigned placeholder; replace with actual extraction timestamp\n \"facts\": [],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "S01" + ], + "required_terminal_result_count": 1 + }, + { + "fixture_id": "raw-response::sensitivity-raw_simple_fact-r03", + "provenance_kind": "exact_empirical_output", + "source_run_id": "sensitivity-raw_simple_fact-r03", + "serialization": "raw", + "sensitivity_cell": "raw_simple_fact", + "seed": 202607150002, + "input_units": [ + { + "unit_id": "S01", + "pool": "simple", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_leaf_count": 1, + "expected_type": "fact" + } + ], + "raw_model_output": "```json\n{\n \"extracted_at\": \"2023-11-15T00:00:00Z\", // Placeholder for current ISO 8601 timestamp\n \"facts\": [],\n \"decisions\": [],\n \"temporal_refs\": []\n}\n```", + "observed_strict_json_parse_success": false, + "observed_finish_reason": "stop", + "expected_unwrap_actions": [ + "strip_markdown_fence", + "strip_line_comments_outside_strings" + ], + "expected_normalization_actions": [], + "expected_normalized_stage1": { + "extracted_at": "2023-11-15T00:00:00Z", + "facts": [], + "decisions": [], + "temporal_refs": [] + }, + "expected_stage1_schema_valid": true, + "expected_unit_results": [ + { + "source_unit_id": "S01", + "observed_mapped_leaf_ids": [], + "observed_primary_status": "missing", + "required_terminal_slot": true, + "required_fallback": true, + "allowed_terminal_statuses": [ + "schema_valid_envelope", + "fail_closed_marker" + ], + "silent_drop_allowed": false + } + ], + "required_fallback_source_unit_ids": [ + "S01" + ], + "required_terminal_result_count": 1 + } + ], + "malformed_leaf_cases": [ + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r01::decisions::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r01::decisions::0", + "source_run_id": "sensitivity-bullets_mixed_N02-r01", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level", + "entity_refs": null, + "decided_at": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported", + "material_clause_omission" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r01::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r01::facts::0", + "source_run_id": "sensitivity-bullets_mixed_N02-r01", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r02::decisions::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r02::decisions::0", + "source_run_id": "sensitivity-bullets_mixed_N02-r02", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level", + "entity_refs": null, + "decided_at": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported", + "split_unit" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r02::decisions::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r02::decisions::1", + "source_run_id": "sensitivity-bullets_mixed_N02-r02", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "because one extract call produces one envelope rather than one packet per input unit", + "entity_refs": null, + "decided_at": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported", + "split_unit" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "because one extract call produces one envelope rather than one packet per input unit" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r02::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r02::facts::0", + "source_run_id": "sensitivity-bullets_mixed_N02-r02", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r03::decisions::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r03::decisions::0", + "source_run_id": "sensitivity-bullets_mixed_N02-r03", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level", + "entity_refs": null, + "decided_at": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported", + "split_unit" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r03::decisions::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r03::decisions::1", + "source_run_id": "sensitivity-bullets_mixed_N02-r03", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "because one extract call produces one envelope rather than one packet per input unit", + "entity_refs": null, + "decided_at": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported", + "split_unit" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "because one extract call produces one envelope rather than one packet per input unit" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N02-r03::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r03::facts::0", + "source_run_id": "sensitivity-bullets_mixed_N02-r03", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r01::decisions::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r01::decisions::0", + "source_run_id": "sensitivity-bullets_mixed_N04-r01", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "decided_at": null, + "entity_refs": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r01::decisions::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r01::decisions::1", + "source_run_id": "sensitivity-bullets_mixed_N04-r01", + "source_unit_ids": [ + "M09" + ], + "field": "decisions", + "raw_leaf": { + "text": "independent Modal variants should run concurrently with spawn.", + "decided_at": "2026-06-22T00:00:00Z", + "entity_refs": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs" + ], + "expected_normalized_leaf": { + "text": "independent Modal variants should run concurrently with spawn.", + "decided_at": "2026-06-22T00:00:00Z" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r01::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r01::facts::0", + "source_run_id": "sensitivity-bullets_mixed_N04-r01", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r02::decisions::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r02::decisions::0", + "source_run_id": "sensitivity-bullets_mixed_N04-r02", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit.", + "decided_at": null, + "entity_refs": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level because one extract call produces one envelope rather than one packet per input unit." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r02::decisions::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r02::decisions::1", + "source_run_id": "sensitivity-bullets_mixed_N04-r02", + "source_unit_ids": [ + "M09" + ], + "field": "decisions", + "raw_leaf": { + "text": "independent Modal variants should run concurrently with spawn.", + "decided_at": "2026-06-22T00:00:00Z", + "entity_refs": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs" + ], + "expected_normalized_leaf": { + "text": "independent Modal variants should run concurrently with spawn.", + "decided_at": "2026-06-22T00:00:00Z" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r02::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r02::facts::0", + "source_run_id": "sensitivity-bullets_mixed_N04-r02", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r03::decisions::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r03::decisions::0", + "source_run_id": "sensitivity-bullets_mixed_N04-r03", + "source_unit_ids": [ + "M05" + ], + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level", + "entity_refs": null, + "decided_at": null + }, + "observed_schema_errors": [ + "entity_refs_not_string_array", + "decided_at_not_string" + ], + "observed_semantic_labels": [ + "source_supported", + "material_clause_omission" + ], + "expected_actions": [ + "omit_null_or_invalid_entity_refs", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level" + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_mixed_N04-r03::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_mixed_N04-r03::facts::0", + "source_run_id": "sensitivity-bullets_mixed_N04-r03", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N02-r01::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N02-r01::facts::0", + "source_run_id": "sensitivity-bullets_simple_N02-r01", + "source_unit_ids": [ + "S01" + ], + "field": "facts", + "raw_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N02-r01::facts::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N02-r01::facts::1", + "source_run_id": "sensitivity-bullets_simple_N02-r01", + "source_unit_ids": [ + "S02" + ], + "field": "facts", + "raw_leaf": { + "text": "The recall package is open source.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The recall package is open source." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N02-r02::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N02-r02::facts::0", + "source_run_id": "sensitivity-bullets_simple_N02-r02", + "source_unit_ids": [ + "S01" + ], + "field": "facts", + "raw_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N02-r02::facts::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N02-r02::facts::1", + "source_run_id": "sensitivity-bullets_simple_N02-r02", + "source_unit_ids": [ + "S02" + ], + "field": "facts", + "raw_leaf": { + "text": "The recall package is open source.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The recall package is open source." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N02-r03::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N02-r03::facts::0", + "source_run_id": "sensitivity-bullets_simple_N02-r03", + "source_unit_ids": [ + "S01" + ], + "field": "facts", + "raw_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N02-r03::facts::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N02-r03::facts::1", + "source_run_id": "sensitivity-bullets_simple_N02-r03", + "source_unit_ids": [ + "S02" + ], + "field": "facts", + "raw_leaf": { + "text": "The recall package is open source.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The recall package is open source." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N04-r01::facts::0", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N04-r01::facts::0", + "source_run_id": "sensitivity-bullets_simple_N04-r01", + "source_unit_ids": [ + "S01" + ], + "field": "facts", + "raw_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "Synapt uses gitgrip to coordinate multiple repositories." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N04-r01::facts::1", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N04-r01::facts::1", + "source_run_id": "sensitivity-bullets_simple_N04-r01", + "source_unit_ids": [ + "S02" + ], + "field": "facts", + "raw_leaf": { + "text": "The recall package is open source.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The recall package is open source." + }, + "semantic_reclassification_allowed": false + }, + { + "fixture_id": "malformed-leaf::sensitivity-bullets_simple_N04-r01::facts::2", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "sensitivity-bullets_simple_N04-r01::facts::2", + "source_run_id": "sensitivity-bullets_simple_N04-r01", + "source_unit_ids": [ + "S03" + ], + "field": "facts", + "raw_leaf": { + "text": "The premium package is proprietary.", + "category": null + }, + "observed_schema_errors": [ + "category_not_string" + ], + "observed_semantic_labels": [ + "source_supported" + ], + "expected_actions": [ + "omit_null_or_non_string_category" + ], + "expected_normalized_leaf": { + "text": "The premium package is proprietary." + }, + "semantic_reclassification_allowed": false + } + ], + "dropped_source_occurrence_cases": [ + { + "fixture_id": "dropped-occurrence::sensitivity-bullets_mixed_N04-r01::M01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-bullets_mixed_N04-r01::M01", + "source_run_id": "sensitivity-bullets_mixed_N04-r01", + "raw_response_fixture_id": "raw-response::sensitivity-bullets_mixed_N04-r01", + "serialization": "numbered_bullets", + "batch_source_unit_ids": [ + "S03", + "M05", + "M09", + "M01" + ], + "dropped_source_unit": { + "source_unit_id": "M01", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-bullets_mixed_N04-r02::M01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-bullets_mixed_N04-r02::M01", + "source_run_id": "sensitivity-bullets_mixed_N04-r02", + "raw_response_fixture_id": "raw-response::sensitivity-bullets_mixed_N04-r02", + "serialization": "numbered_bullets", + "batch_source_unit_ids": [ + "S03", + "M05", + "M09", + "M01" + ], + "dropped_source_unit": { + "source_unit_id": "M01", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-bullets_mixed_N04-r03::M01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-bullets_mixed_N04-r03::M01", + "source_run_id": "sensitivity-bullets_mixed_N04-r03", + "raw_response_fixture_id": "raw-response::sensitivity-bullets_mixed_N04-r03", + "serialization": "numbered_bullets", + "batch_source_unit_ids": [ + "S03", + "M05", + "M09", + "M01" + ], + "dropped_source_unit": { + "source_unit_id": "M01", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-bullets_simple_N04-r01::S04", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-bullets_simple_N04-r01::S04", + "source_run_id": "sensitivity-bullets_simple_N04-r01", + "raw_response_fixture_id": "raw-response::sensitivity-bullets_simple_N04-r01", + "serialization": "numbered_bullets", + "batch_source_unit_ids": [ + "S01", + "S02", + "S03", + "S04" + ], + "dropped_source_unit": { + "source_unit_id": "S04", + "text": "The extract library emits SynaptExtraction documents.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-raw_multi_clause-r01::M01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-raw_multi_clause-r01::M01", + "source_run_id": "sensitivity-raw_multi_clause-r01", + "raw_response_fixture_id": "raw-response::sensitivity-raw_multi_clause-r01", + "serialization": "raw", + "batch_source_unit_ids": [ + "M01" + ], + "dropped_source_unit": { + "source_unit_id": "M01", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-raw_multi_clause-r02::M01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-raw_multi_clause-r02::M01", + "source_run_id": "sensitivity-raw_multi_clause-r02", + "raw_response_fixture_id": "raw-response::sensitivity-raw_multi_clause-r02", + "serialization": "raw", + "batch_source_unit_ids": [ + "M01" + ], + "dropped_source_unit": { + "source_unit_id": "M01", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-raw_multi_clause-r03::M01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-raw_multi_clause-r03::M01", + "source_run_id": "sensitivity-raw_multi_clause-r03", + "raw_response_fixture_id": "raw-response::sensitivity-raw_multi_clause-r03", + "serialization": "raw", + "batch_source_unit_ids": [ + "M01" + ], + "dropped_source_unit": { + "source_unit_id": "M01", + "text": "The extract builder receives clean knowledge units, while recall remains responsible for identifying those units and reconciling them afterward.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-raw_simple_fact-r01::S01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-raw_simple_fact-r01::S01", + "source_run_id": "sensitivity-raw_simple_fact-r01", + "raw_response_fixture_id": "raw-response::sensitivity-raw_simple_fact-r01", + "serialization": "raw", + "batch_source_unit_ids": [ + "S01" + ], + "dropped_source_unit": { + "source_unit_id": "S01", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-raw_simple_fact-r02::S01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-raw_simple_fact-r02::S01", + "source_run_id": "sensitivity-raw_simple_fact-r02", + "raw_response_fixture_id": "raw-response::sensitivity-raw_simple_fact-r02", + "serialization": "raw", + "batch_source_unit_ids": [ + "S01" + ], + "dropped_source_unit": { + "source_unit_id": "S01", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + }, + { + "fixture_id": "dropped-occurrence::sensitivity-raw_simple_fact-r03::S01", + "provenance_kind": "exact_empirical_drop", + "source_occurrence_id": "sensitivity-raw_simple_fact-r03::S01", + "source_run_id": "sensitivity-raw_simple_fact-r03", + "raw_response_fixture_id": "raw-response::sensitivity-raw_simple_fact-r03", + "serialization": "raw", + "batch_source_unit_ids": [ + "S01" + ], + "dropped_source_unit": { + "source_unit_id": "S01", + "text": "Synapt uses gitgrip to coordinate multiple repositories.", + "expected_type": "fact" + }, + "observed_mapped_leaf_ids": [], + "expected_contract_behavior": { + "source_attribution_preserved": true, + "per_unit_fallback_invoked": true, + "silent_drop_allowed": false, + "terminal_statuses_allowed": [ + "schema_valid_envelope", + "fail_closed_marker" + ] + } + } + ], + "unknown_key_leaf_cases": [ + { + "fixture_id": "unknown-key-leaf::matrix-mixed_category-N16-r03::decisions::4", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "matrix-mixed_category-N16-r03::decisions::4", + "source_run_id": "matrix-mixed_category-N16-r03", + "source_unit_ids": [ + "M10" + ], + "field": "decisions", + "raw_leaf": { + "text": "On July 1, 2026, per-epoch checkpointing became mandatory for long Modal jobs after repeated preemptions.", + "decided_at": "2026-07-01", + "context": "mandatory rule" + }, + "observed_schema_errors": [ + "extra_keys:context" + ], + "expected_actions": [ + "drop_unknown_key:context" + ], + "expected_normalized_leaf": { + "text": "On July 1, 2026, per-epoch checkpointing became mandatory for long Modal jobs after repeated preemptions.", + "decided_at": "2026-07-01" + } + }, + { + "fixture_id": "unknown-key-leaf::matrix-multi_clause-N16-r03::facts::8", + "provenance_kind": "exact_empirical_leaf", + "source_leaf_id": "matrix-multi_clause-N16-r03::facts::8", + "source_run_id": "matrix-multi_clause-N16-r03", + "source_unit_ids": [ + "M10" + ], + "field": "facts", + "raw_leaf": { + "text": "On July 1, 2026, per-epoch checkpointing became mandatory for long Modal jobs after repeated preemptions.", + "type": "range", + "resolved": "2026-07-01T00:00:00Z", + "resolved_end": "2026-07-13T00:00:00Z", + "context": "Mandatory after preemptions" + }, + "observed_schema_errors": [ + "extra_keys:context,resolved,resolved_end,type" + ], + "expected_actions": [ + "drop_unknown_key:context", + "drop_unknown_key:resolved", + "drop_unknown_key:resolved_end", + "drop_unknown_key:type" + ], + "expected_normalized_leaf": { + "text": "On July 1, 2026, per-epoch checkpointing became mandatory for long Modal jobs after repeated preemptions." + } + } + ], + "temporal_shape_cases": [ + { + "fixture_id": "temporal-shape::sensitivity-bullets_mixed_N04-r01::0", + "provenance_kind": "exact_empirical_temporal_leaf", + "source_run_id": "sensitivity-bullets_mixed_N04-r01", + "temporal_index": 0, + "raw_temporal_ref": { + "raw": "June 22, 2026", + "type": "point", + "resolved": "2026-06-22T00:00:00Z", + "context": "decision reference" + }, + "observed_prompt_schema_conflict": "The builder prompt requests type/context/resolved_end, but the Stage-1 schema permits only raw and optional resolved.", + "expected_actions": [ + "drop_unknown_key:context", + "drop_unknown_key:type" + ], + "expected_normalized_temporal_ref": { + "raw": "June 22, 2026", + "resolved": "2026-06-22T00:00:00Z" + } + }, + { + "fixture_id": "temporal-shape::sensitivity-bullets_mixed_N04-r02::0", + "provenance_kind": "exact_empirical_temporal_leaf", + "source_run_id": "sensitivity-bullets_mixed_N04-r02", + "temporal_index": 0, + "raw_temporal_ref": { + "raw": "June 22, 2026", + "type": "point", + "resolved": "2026-06-22T00:00:00Z", + "context": "decision reference" + }, + "observed_prompt_schema_conflict": "The builder prompt requests type/context/resolved_end, but the Stage-1 schema permits only raw and optional resolved.", + "expected_actions": [ + "drop_unknown_key:context", + "drop_unknown_key:type" + ], + "expected_normalized_temporal_ref": { + "raw": "June 22, 2026", + "resolved": "2026-06-22T00:00:00Z" + } + }, + { + "fixture_id": "temporal-shape::sensitivity-bullets_mixed_N04-r03::0", + "provenance_kind": "exact_empirical_temporal_leaf", + "source_run_id": "sensitivity-bullets_mixed_N04-r03", + "temporal_index": 0, + "raw_temporal_ref": { + "raw": "June 22, 2026", + "type": "point", + "resolved": "2026-06-22", + "context": "Establishment of concurrent spawn for Modal variants" + }, + "observed_prompt_schema_conflict": "The builder prompt requests type/context/resolved_end, but the Stage-1 schema permits only raw and optional resolved.", + "expected_actions": [ + "drop_unknown_key:context", + "drop_unknown_key:type" + ], + "expected_normalized_temporal_ref": { + "raw": "June 22, 2026", + "resolved": "2026-06-22" + } + } + ], + "contract_derived_cases": [ + { + "fixture_id": "contract-derived::entity-refs-scalar-to-array", + "provenance_kind": "contract_derived_sibling_of_empirical_null_failure", + "source_leaf_id": "sensitivity-bullets_mixed_N02-r01::decisions::0", + "empirical_difference": "The run emitted entity_refs=null, never a scalar. This fixture changes only that value to a string because scalar-to-array is in the shared contract.", + "field": "decisions", + "raw_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level", + "entity_refs": "facts-and-decisions leaf level", + "decided_at": null + }, + "expected_actions": [ + "coerce_entity_refs_scalar_to_array", + "omit_null_or_non_string_decided_at" + ], + "expected_normalized_leaf": { + "text": "measure fidelity at the facts-and-decisions leaf level", + "entity_refs": [ + "facts-and-decisions leaf level" + ] + }, + "must_not_be_reported_as_empirically_observed": true + } + ] +} diff --git a/tests/python/test_conformance.py b/tests/python/test_conformance.py index be708b9..b40f03e 100644 --- a/tests/python/test_conformance.py +++ b/tests/python/test_conformance.py @@ -8,9 +8,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "python" / "src")) -from synapt_extract.finalize import finalize_extraction, FinalizeContext -from synapt_extract.prompt import build_extraction_prompt, resolve_capabilities -from synapt_extract.validate import validate_extraction +from synapt.extract.finalize import finalize_extraction, FinalizeContext +from synapt.extract.prompt import build_extraction_prompt, resolve_capabilities +from synapt.extract.validate import validate_extraction FIXTURES_DIR = Path(__file__).resolve().parents[1] / "conformance" diff --git a/tests/python/test_extract.py b/tests/python/test_extract.py index 825201e..90fba3b 100644 --- a/tests/python/test_extract.py +++ b/tests/python/test_extract.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "python" / "src")) -from synapt_extract import create_extraction_builder, extract, extract_openai +from synapt.extract import create_extraction_builder, extract, extract_openai SAMPLE_TEXT = ( diff --git a/tests/python/test_extract_batch.py b/tests/python/test_extract_batch.py new file mode 100644 index 0000000..8be040c --- /dev/null +++ b/tests/python/test_extract_batch.py @@ -0,0 +1,581 @@ +"""Contract tests for reliable, source-attributed batch extraction.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import sys +from copy import deepcopy +from pathlib import Path +from typing import get_args + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "python" / "src")) + +from synapt.extract import ( + BatchFailureReason, + BatchUnit, + extract_batch, + profile_capabilities, + validate_extraction, +) +from synapt.extract.batch import _coerce_shape, _strip_output_hygiene + + +RECALL_CAPABILITIES = ["facts", "decisions", "temporal_refs"] +PRODUCED_BY = "mlx://mlx-community/Ministral-3-3B-Instruct-2512-4bit" +EXTRACTED_AT = "2026-07-13T10:00:00Z" +FIXTURE_SHA256 = "9b183f18ab5116cfb1f5ee67d0e99cd5af3fb7f7b99d649b1d58821f9e7489f1" +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "extract-batch-real-failures-v1.json" +FIXTURE_BYTES = FIXTURE_PATH.read_bytes() +FIXTURES = json.loads(FIXTURE_BYTES) +USE_STANDARD_DEFAULT = object() + + +def _stage1(*, facts=None, decisions=None, temporal_refs=None, **extra): + return { + "extracted_at": EXTRACTED_AT, + "facts": [] if facts is None else facts, + "decisions": [] if decisions is None else decisions, + "temporal_refs": [] if temporal_refs is None else temporal_refs, + **extra, + } + + +def _request_prompt(request): + return request["prompt"] + + +def _model_visible_text(request): + return "\n".join(message["content"] for message in request["messages"]) + + +def _run_batch(units, infer, *, capabilities=RECALL_CAPABILITIES, **options): + kwargs = { + "infer": infer, + "produced_by": PRODUCED_BY, + **options, + } + if capabilities is not USE_STANDARD_DEFAULT: + kwargs["capabilities"] = capabilities + return asyncio.run(extract_batch(units, **kwargs)) + + +def _response_for_prompt(request, responses_by_source): + prompt = _request_prompt(request) + matching = [source for source in responses_by_source if source in prompt] + if len(matching) > 1: + # A batch-first implementation may choose this path. Force its per-unit + # fallback without prescribing the primary batching strategy. + return "primary batch requires per-unit fallback" + assert len(matching) == 1, f"request did not contain a known source unit: {prompt}" + return responses_by_source[matching[0]] + + +def _assert_success(output, source_unit_id): + assert output.source_unit_id == source_unit_id + assert output.extraction is not None + assert validate_extraction(output.extraction).valid + + +def _assert_failure(output, source_unit_id, reason): + assert output.source_unit_id == source_unit_id + assert output.status == "failed" + assert output.reason == reason + + +def _fixture_case(group, fixture_id): + return next(case for case in FIXTURES[group] if case["fixture_id"] == fixture_id) + + +def test_real_failure_fixture_pack_is_sha_pinned_and_complete(): + assert hashlib.sha256(FIXTURE_BYTES).hexdigest() == FIXTURE_SHA256 + assert len(FIXTURES["raw_response_cases"]) == 21 + assert len(FIXTURES["malformed_leaf_cases"]) == 25 + assert len(FIXTURES["dropped_source_occurrence_cases"]) == 10 + assert len(FIXTURES["unknown_key_leaf_cases"]) == 2 + assert len(FIXTURES["temporal_shape_cases"]) == 3 + assert len(FIXTURES["contract_derived_cases"]) == 1 + + +@pytest.mark.parametrize( + "case", + FIXTURES["raw_response_cases"], + ids=lambda case: case["fixture_id"], +) +def test_real_raw_responses_strip_hygiene_and_normalize_exactly(case): + cleaned = _strip_output_hygiene(case["raw_model_output"]) + normalized = _coerce_shape(json.loads(cleaned), RECALL_CAPABILITIES) + + assert normalized == case["expected_normalized_stage1"] + + +def test_comment_hygiene_preserves_double_slashes_inside_json_strings(): + raw = '''```json +{ + "extracted_at": "2026-07-13T10:00:00Z", // remove this comment + "facts": [{"text": "Schema: https://synapt.dev/schemas/extract/v1.json"}], + "decisions": [], + "temporal_refs": [] +} +```''' + + parsed = json.loads(_strip_output_hygiene(raw)) + + assert parsed["facts"][0]["text"] == "Schema: https://synapt.dev/schemas/extract/v1.json" + + +@pytest.mark.parametrize( + "case", + FIXTURES["unknown_key_leaf_cases"], + ids=lambda case: case["fixture_id"], +) +def test_real_unknown_leaf_keys_are_dropped(case): + stage1 = _stage1() + stage1[case["field"]] = [deepcopy(case["raw_leaf"])] + + normalized = _coerce_shape(stage1, RECALL_CAPABILITIES) + + assert normalized[case["field"]] == [case["expected_normalized_leaf"]] + + +@pytest.mark.parametrize( + "case", + FIXTURES["temporal_shape_cases"], + ids=lambda case: case["fixture_id"], +) +def test_temporal_prompt_schema_conflict_is_explicitly_normalized(case): + stage1 = _stage1(temporal_refs=[deepcopy(case["raw_temporal_ref"])]) + + normalized = _coerce_shape(stage1, RECALL_CAPABILITIES) + + assert normalized["temporal_refs"] == [case["expected_normalized_temporal_ref"]] + assert set(normalized["temporal_refs"][0]) <= {"raw", "resolved"} + + +def test_role_and_resolved_end_survive_coercion_at_base_capability_tier(): + """THE root cause this whole fix chain traces back to (config/design/extract-temporal- + role-2026-07-14.md): role + resolved_end are BASE-tier now, not gated behind + temporal_classes — _coerce_shape's whitelist is schema-driven (build_extraction_schema), + so this is a pure consequence of the builder.py fix, not separate coercion code (VERIFIED + empirically before this test existed, ad hoc; formalized here as a permanent regression + guard). Uses ONLY RECALL_CAPABILITIES ("temporal_refs", no "temporal_classes") — the exact + capability set recall's B1 requests.""" + stage1 = _stage1(temporal_refs=[{ + "raw": "March to April 2026", "role": "range", + "resolved": "2026-03-01", "resolved_end": "2026-04-30", + }]) + + normalized = _coerce_shape(stage1, RECALL_CAPABILITIES) + + assert normalized["temporal_refs"][0] == { + "raw": "March to April 2026", "role": "range", + "resolved": "2026-03-01", "resolved_end": "2026-04-30", + } + + +def test_type_and_context_still_stripped_at_base_capability_tier(): + """Negative control: type/context remain temporal_classes-gated (non-load-bearing extras + once role carries the direction signal) — confirms the fix is precisely scoped to + role+resolved_end, not an accidental full unlock of every temporal_classes field.""" + stage1 = _stage1(temporal_refs=[{ + "raw": "expires April 30", "role": "expiry", "type": "point", + "resolved": "2026-04-30", "context": "API key", + }]) + + normalized = _coerce_shape(stage1, RECALL_CAPABILITIES) + + assert normalized["temporal_refs"][0] == { + "raw": "expires April 30", "role": "expiry", "resolved": "2026-04-30", + } + + +def test_entity_refs_scalar_is_coerced_in_scope_and_dropped_out_of_scope(): + case = FIXTURES["contract_derived_cases"][0] + assert case["must_not_be_reported_as_empirically_observed"] is True + + in_scope = _coerce_shape( + _stage1(decisions=[deepcopy(case["raw_leaf"])]), + ["entities", "decisions"], + ) + out_of_scope = _coerce_shape( + _stage1(decisions=[deepcopy(case["raw_leaf"])]), + RECALL_CAPABILITIES, + ) + + assert in_scope["decisions"] == [case["expected_normalized_leaf"]] + assert out_of_scope["decisions"] == [ + {"text": case["expected_normalized_leaf"]["text"]} + ] + + +def test_extract_batch_runs_a_real_recorded_failure_to_strict_validity(): + case = _fixture_case( + "raw_response_cases", + "raw-response::sensitivity-raw_decision-r01", + ) + source = case["input_units"][0] + requests = [] + + def infer(request): + requests.append(request) + return case["raw_model_output"] + + outputs = _run_batch( + [BatchUnit(id=source["unit_id"], text=source["text"])], + infer, + ) + + assert len(outputs) == 1 + _assert_success(outputs[0], source["unit_id"]) + for field in ("facts", "decisions", "temporal_refs"): + assert outputs[0].extraction[field] == case["expected_normalized_stage1"][field] + assert all(source["unit_id"] not in _model_visible_text(request) for request in requests) + assert all("[UNIT" not in _model_visible_text(request) for request in requests) + + +def test_extract_batch_uses_per_call_capabilities_with_per_unit_overrides(): + units = [ + BatchUnit( + id="fact-only", + text="The extract library emits SynaptExtraction documents.", + capabilities=["facts"], + ), + BatchUnit( + id="decision-only", + text="The team decided to keep unit boundaries out of band.", + capabilities=["decisions"], + ), + ] + responses = { + units[0].text: json.dumps(_stage1(facts=[{"text": units[0].text}])), + units[1].text: json.dumps(_stage1(decisions=[{"text": units[1].text}])), + } + seen_capabilities = {} + + def infer(request): + prompt = _request_prompt(request) + for unit in units: + if unit.text in prompt: + seen_capabilities[unit.id] = request["capabilities"] + return _response_for_prompt(request, responses) + + outputs = _run_batch(units, infer, capabilities=["facts", "decisions"]) + + assert seen_capabilities == { + "fact-only": ["facts"], + "decision-only": ["decisions"], + } + assert [output.source_unit_id for output in outputs] == ["fact-only", "decision-only"] + assert len(outputs) == len(units) + _assert_success(outputs[0], "fact-only") + _assert_success(outputs[1], "decision-only") + + +def test_extract_batch_threads_unit_date_as_temporal_resolution_anchor(): + """config/design/extract-temporal-role-2026-07-14.md 'Temporal RESOLUTION needs the + source date': each unit's SOURCE date threads into Stage-1 as the resolution anchor for + partial/relative dates. Uses a NON-2026 source date (Sentinel's explicit ask — every prior + temporal test used 2026, which masked exactly this class of bug).""" + unit = BatchUnit(id="anchored", text="the API key expires April 30", date="2025-03-01") + seen_prompts = [] + + def infer(request): + seen_prompts.append(_request_prompt(request)) + return json.dumps(_stage1(temporal_refs=[ + {"raw": "expires April 30", "role": "expiry", "resolved": "2025-04-30"}, + ])) + + outputs = _run_batch([unit], infer, capabilities=["temporal_refs"]) + + assert len(seen_prompts) == 1 + assert "2025-03-01" in seen_prompts[0] # the anchor reached the model-visible prompt + _assert_success(outputs[0], "anchored") + + +def test_extract_batch_replicates_sentinels_wrong_year_scenario_end_to_end(): + """THE capstone: role (direction) + resolution (source-date anchor) working TOGETHER + through a REAL extract_batch call, replicating Sentinel's exact real-path finding + (config/design/extract-temporal-role-2026-07-14.md 'Temporal RESOLUTION needs the source + date') — a 2025-03-01-sourced unit with "API key expires April 30" must NOT silently + resolve to 2026 (the c791018 duct-tape bug this whole fix chain traces back to). This test + proves the CONTRACT end-to-end: given a correctly-anchored+classified model response, the + persisted envelope carries role="expiry" and the ANCHORED year — not whether a real model + reliably produces that response (a model-quality question for Phase-C), but that nothing + in the wiring between the anchor and the final envelope silently discards or corrupts it.""" + unit = BatchUnit( + id="clu:0:done:0", text="the API key expires April 30", date="2025-03-01", + ) + + def infer(request): + assert "2025-03-01" in _request_prompt(request) # the anchor reached the model + return json.dumps(_stage1(temporal_refs=[ + {"raw": "expires April 30", "role": "expiry", "resolved": "2025-04-30"}, + ])) + + outputs = _run_batch([unit], infer, capabilities=["temporal_refs"]) + + _assert_success(outputs[0], "clu:0:done:0") + ref = outputs[0].extraction["temporal_refs"][0] + assert ref["role"] == "expiry" # direction preserved through coercion + validation + assert ref["resolved"] == "2025-04-30" # ANCHORED year, not 2026 (Sentinel's bug) + + +def test_extract_batch_unit_without_date_omits_resolution_anchor_gracefully(): + """Backward compat: a BatchUnit with no date (existing callers, or a candidate whose + source entry has no timestamp) must not crash — extract_batch degrades gracefully rather + than injecting a bogus anchor.""" + unit = BatchUnit(id="no-date", text="a fact with no date anchor") + + def infer(request): + return json.dumps(_stage1(facts=[{"text": unit.text}])) + + outputs = _run_batch([unit], infer, capabilities=["facts"]) + _assert_success(outputs[0], "no-date") + + +def test_extract_batch_unit_date_is_per_unit_not_shared_across_the_batch(): + """Two units in the SAME batch with DIFFERENT source dates — each unit's OWN prompt must + carry its OWN anchor, not bleed the other unit's date (per-unit call shape, v1).""" + units = [ + BatchUnit(id="unit-2024", text="fact from an older entry", date="2024-06-01"), + BatchUnit(id="unit-2026", text="fact from a newer entry", date="2026-01-15"), + ] + seen_by_unit = {} + + def infer(request): + prompt = _request_prompt(request) + for unit in units: + if unit.text in prompt: + seen_by_unit[unit.id] = prompt + return json.dumps(_stage1(facts=[{"text": "a fact"}])) + + _run_batch(units, infer, capabilities=["facts"]) + + assert "2024-06-01" in seen_by_unit["unit-2024"] + assert "2026-01-15" not in seen_by_unit["unit-2024"] + assert "2026-01-15" in seen_by_unit["unit-2026"] + assert "2024-06-01" not in seen_by_unit["unit-2026"] + + +def test_extract_batch_uses_the_standard_profile_when_call_capabilities_are_omitted(): + unit = BatchUnit(id="standard-default", text="The standard profile remains the default.") + seen = [] + standard_stage1 = { + "extracted_at": EXTRACTED_AT, + "entities": [], + "goals": [], + "themes": [], + "summary": unit.text, + "sentiment": "neutral", + "facts": [{"text": unit.text}], + "temporal_refs": [], + } + + def infer(request): + seen.append(request["capabilities"]) + return json.dumps(standard_stage1) + + outputs = _run_batch( + [unit], + infer, + capabilities=USE_STANDARD_DEFAULT, + ) + + assert seen == [profile_capabilities("standard")] + _assert_success(outputs[0], unit.id) + + +def test_dropped_real_output_retries_through_the_inference_seam(): + case = _fixture_case( + "raw_response_cases", + "raw-response::sensitivity-raw_multi_clause-r01", + ) + source = case["input_units"][0] + completions = [ + case["raw_model_output"], + json.dumps(_stage1(facts=[{"text": source["text"]}])), + ] + calls = 0 + + def infer(_request): + nonlocal calls + response = completions[min(calls, len(completions) - 1)] + calls += 1 + return response + + outputs = _run_batch( + [BatchUnit(id=source["unit_id"], text=source["text"])], + infer, + ) + + assert calls >= 2 + assert len(outputs) == 1 + _assert_success(outputs[0], source["unit_id"]) + assert outputs[0].extraction["facts"] == [{"text": source["text"]}] + + +@pytest.mark.parametrize( + ("completion", "reason"), + [ + ("not valid JSON", "unparseable"), + (json.dumps(_stage1(facts=[{"text": 42}])), "schema_invalid"), + ], +) +def test_terminal_failures_preserve_the_unit_slot(completion, reason): + unit = BatchUnit(id=f"terminal-{reason}", text="This source unit remains attributable.") + + outputs = _run_batch([unit], lambda _request: completion) + + assert len(outputs) == 1 + _assert_failure(outputs[0], unit.id, reason) + + +@pytest.mark.parametrize( + "case", + FIXTURES["dropped_source_occurrence_cases"], + ids=lambda case: case["fixture_id"], +) +def test_real_dropped_source_occurrences_never_disappear(case): + source = case["dropped_source_unit"] + unit = BatchUnit(id=source["source_unit_id"], text=source["text"]) + + outputs = _run_batch([unit], lambda _request: json.dumps(_stage1())) + + assert len(outputs) == 1 + _assert_failure(outputs[0], unit.id, "dropped") + + +def test_failure_reason_contract_includes_merged_without_widening(): + assert set(get_args(BatchFailureReason)) == { + "unparseable", + "schema_invalid", + "dropped", + "merged", + } + + +def test_one_bad_unit_never_voids_its_neighbors(): + units = [ + BatchUnit(id="good-before", text="The first durable fact is grounded."), + BatchUnit(id="bad-middle", text="The malformed source remains attributable."), + BatchUnit(id="good-after", text="The final durable fact is grounded."), + ] + responses = { + units[0].text: json.dumps(_stage1(facts=[{"text": units[0].text}])), + units[1].text: json.dumps(_stage1(facts=[{"text": {"not": "a string"}}])), + units[2].text: json.dumps(_stage1(facts=[{"text": units[2].text}])), + } + + outputs = _run_batch( + units, + lambda request: _response_for_prompt(request, responses), + ) + + assert len(outputs) == len(units) + assert [output.source_unit_id for output in outputs] == [unit.id for unit in units] + _assert_success(outputs[0], "good-before") + _assert_failure(outputs[1], "bad-middle", "schema_invalid") + _assert_success(outputs[2], "good-after") + + +def test_duplicate_unit_ids_are_rejected_before_inference(): + calls = 0 + + def infer(_request): + nonlocal calls + calls += 1 + return json.dumps(_stage1()) + + with pytest.raises(ValueError, match="duplicate.*id|id.*unique"): + _run_batch( + [ + BatchUnit(id="same", text="First source."), + BatchUnit(id="same", text="Second source."), + ], + infer, + ) + + assert calls == 0 + + +def test_empty_input_is_a_noop(): + outputs = _run_batch( + [], + lambda _request: pytest.fail("empty batches must not invoke inference"), + ) + + assert outputs == [] + + +# --- Fidelity-gate regression locks (Sentinel, extract#30 re-gate) ------------- + +def test_infer_exception_in_one_unit_never_voids_the_batch(): + """HIGH-1: an infer() exception must be contained to its own unit — neighbours + still produce their slots and the count-invariant holds (no output produced for + the failing unit → terminal "dropped" after retry).""" + units = [ + BatchUnit(id="good-before", text="The first durable fact is grounded."), + BatchUnit(id="raises", text="This unit's inference raises."), + BatchUnit(id="good-after", text="The final durable fact is grounded."), + ] + responses = { + units[0].text: json.dumps(_stage1(facts=[{"text": units[0].text}])), + units[2].text: json.dumps(_stage1(facts=[{"text": units[2].text}])), + } + + def infer(request): + if units[1].text in _request_prompt(request): + raise RuntimeError("inference exploded") + return _response_for_prompt(request, responses) + + outputs = _run_batch(units, infer) + + assert len(outputs) == len(units) + assert [output.source_unit_id for output in outputs] == [unit.id for unit in units] + _assert_success(outputs[0], "good-before") + _assert_failure(outputs[1], "raises", "dropped") + _assert_success(outputs[2], "good-after") + + +@pytest.mark.parametrize( + ("label", "malformed"), + [ + ("facts_null", _stage1(facts=[None])), + ("facts_null_beside_valid", _stage1(facts=[None, {"text": "A grounded durable fact."}])), + ("decisions_non_dict", _stage1(decisions=[42])), + ("temporal_non_dict", _stage1(temporal_refs=["tomorrow"])), + ], +) +def test_non_dict_leaves_reach_strict_validation(label, malformed): + """HIGH-2: a non-object leaf must not be silently deleted — it reaches strict + validation and fails its unit (schema_invalid), even beside a valid sibling.""" + unit = BatchUnit(id=f"nondict-{label}", text="This source unit stays attributable.") + + outputs = _run_batch([unit], lambda _request: json.dumps(malformed)) + + assert len(outputs) == 1 + _assert_failure(outputs[0], unit.id, "schema_invalid") + + +def test_valid_metadata_only_extraction_is_not_dropped(): + """HIGH-3: extract_batch is a GENERAL primitive (Q3) — an entities-only (array) + or summary-only (scalar) extraction is real content for its requested capability + set, not the empty "dropped" mode.""" + entities_unit = BatchUnit(id="entities-only", text="Synapt is an organization.") + entities_out = _run_batch( + [entities_unit], + lambda _request: json.dumps(_stage1(entities=[{"name": "Synapt", "type": "org"}])), + capabilities=["entities"], + ) + _assert_success(entities_out[0], "entities-only") + + summary_unit = BatchUnit(id="summary-only", text="A paragraph worth summarizing.") + summary_out = _run_batch( + [summary_unit], + lambda _request: json.dumps(_stage1(summary="A concise summary of the source.")), + capabilities=["summary"], + ) + _assert_success(summary_out[0], "summary-only") diff --git a/tests/python/test_finalize.py b/tests/python/test_finalize.py index b15fe5e..1c9d916 100644 --- a/tests/python/test_finalize.py +++ b/tests/python/test_finalize.py @@ -9,7 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "python" / "src")) -from synapt_extract.finalize import finalize_extraction, FinalizeContext +from synapt.extract.finalize import finalize_extraction, FinalizeContext def _llm_output(**overrides): @@ -371,6 +371,25 @@ def test_detects_temporal_classes(self): assert "temporal_refs" in caps assert "temporal_classes" in caps + def test_resolved_end_alone_does_not_imply_temporal_classes(self): + """role + resolved_end are BASE-tier (config/design/extract-temporal-role-2026-07-14.md) + — a range-role ref can legitimately carry resolved_end WITHOUT the temporal_classes + capability ever being requested/exercised. The old heuristic (`type is not None OR + resolved_end is not None`) would have mislabeled this as having used temporal_classes; + only `type`'s presence (still temporal_classes-gated) should trigger detection now.""" + result = finalize_extraction( + _llm_output(temporal_refs=[{ + "raw": "March to April 2026", + "role": "range", + "resolved": "2026-03-01", + "resolved_end": "2026-04-30", + }]), + FinalizeContext(produced_by="test://model"), + ) + caps = result.extraction["capabilities"] + assert "temporal_refs" in caps + assert "temporal_classes" not in caps + class TestStage3Warnings: diff --git a/tests/python/test_prompt.py b/tests/python/test_prompt.py index 6ee6b9c..43dd25e 100644 --- a/tests/python/test_prompt.py +++ b/tests/python/test_prompt.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "python" / "src")) -from synapt_extract import ( +from synapt.extract import ( ExtractionBuilder, build_finalized_extraction_schema, build_extraction_response_format, @@ -261,6 +261,27 @@ def test_temporal_refs_fragment_present(self): result = build_extraction_prompt(SAMPLE_TEXT, capabilities=["temporal_refs"]) assert '"temporal_refs"' in result + def test_temporal_refs_role_instructions_present(self): + """config/design/extract-temporal-role-2026-07-14.md: the Stage-1 prompt classifies + each temporal ref's validity role, with the 5 enum values named.""" + result = build_extraction_prompt(SAMPLE_TEXT, capabilities=["temporal_refs"]) + assert '"role"' in result + for role in ("effective", "expiry", "range", "superseded", "point"): + assert role in result + + def test_temporal_refs_omits_resolve_instruction_when_no_date_given(self): + """Regression guard: WITHOUT a date, the fragment must NOT render the literal string + "None" as a resolution instruction (the {{date}} template var was previously + unconditional — an absent date rendered as "Resolve relative dates using: None.", + actively misleading the model). Found while building the resolution-anchor fix.""" + result = build_extraction_prompt(SAMPLE_TEXT, capabilities=["temporal_refs"]) + assert "using: None" not in result + assert "None." not in result + + def test_temporal_refs_includes_resolve_instruction_when_date_given(self): + result = build_extraction_prompt(SAMPLE_TEXT, capabilities=["temporal_refs"], date="2025-03-01") + assert "Resolve relative dates using: 2025-03-01" in result + def test_relations_fragment_present(self): result = build_extraction_prompt( SAMPLE_TEXT, @@ -433,7 +454,7 @@ def test_full_profile_is_superset_of_standard(self): assert standard.issubset(full) def test_full_profile_includes_all_capabilities(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.schema import EXTRACTION_CAPABILITIES profiles_dir = Path(__file__).resolve().parents[2] / "prompts" / "profiles" data = json.loads((profiles_dir / "full.json").read_text()) caps = set(data["capabilities"]) @@ -443,20 +464,20 @@ def test_full_profile_includes_all_capabilities(self): class TestRegistryConsistency: def test_capability_registry_covers_schema_capabilities_in_canonical_order(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES - from synapt_extract.prompt import CAPABILITY_REGISTRY, CANONICAL_ORDER + from synapt.extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.prompt import CAPABILITY_REGISTRY, CANONICAL_ORDER assert [definition["name"] for definition in CAPABILITY_REGISTRY["capabilities"]] == CANONICAL_ORDER assert set(CANONICAL_ORDER) == EXTRACTION_CAPABILITIES def test_capability_registry_profiles_match_legacy_profile_files(self): - from synapt_extract.prompt import CAPABILITY_REGISTRY + from synapt.extract.prompt import CAPABILITY_REGISTRY profiles_dir = Path(__file__).resolve().parents[2] / "prompts" / "profiles" for profile in ("minimal", "standard", "full"): file_profile = json.loads((profiles_dir / f"{profile}.json").read_text())["capabilities"] assert CAPABILITY_REGISTRY["profiles"][profile] == file_profile def test_capability_registry_exposes_embedding_inputs(self): - from synapt_extract.prompt import STANDARD_EMBEDDING_INPUTS, capability_embedding_input + from synapt.extract.prompt import STANDARD_EMBEDDING_INPUTS, capability_embedding_input assert capability_embedding_input("entities") == "entities" assert capability_embedding_input("entity_state") == "entities" assert capability_embedding_input("structured_sentiment") == "sentiment" @@ -477,13 +498,13 @@ def test_capability_registry_exposes_embedding_inputs(self): ] def test_every_capability_has_fragment_file(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.schema import EXTRACTION_CAPABILITIES prompts_dir = Path(__file__).resolve().parents[2] / "prompts" / "v1" for cap in EXTRACTION_CAPABILITIES: assert (prompts_dir / f"{cap}.txt").exists(), f"missing fragment for {cap}" def test_every_fragment_is_valid_capability(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.schema import EXTRACTION_CAPABILITIES prompts_dir = Path(__file__).resolve().parents[2] / "prompts" / "v1" for txt_file in prompts_dir.glob("*.txt"): name = txt_file.stem @@ -492,25 +513,25 @@ def test_every_fragment_is_valid_capability(self): assert name in EXTRACTION_CAPABILITIES, f"orphan fragment: {name}" def test_canonical_order_covers_all_capabilities(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES - from synapt_extract.prompt import CANONICAL_ORDER + from synapt.extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.prompt import CANONICAL_ORDER assert set(CANONICAL_ORDER) == EXTRACTION_CAPABILITIES def test_canonical_order_has_no_duplicates(self): - from synapt_extract.prompt import CANONICAL_ORDER + from synapt.extract.prompt import CANONICAL_ORDER assert len(CANONICAL_ORDER) == len(set(CANONICAL_ORDER)) def test_capability_deps_reference_valid_capabilities(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES - from synapt_extract.prompt import CAPABILITY_DEPS + from synapt.extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.prompt import CAPABILITY_DEPS for cap, deps in CAPABILITY_DEPS.items(): assert cap in EXTRACTION_CAPABILITIES, f"dep key {cap} not a valid capability" for dep in deps: assert dep in EXTRACTION_CAPABILITIES, f"dep {dep} (from {cap}) not valid" def test_capability_rules_reference_valid_capabilities(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES - from synapt_extract.prompt import CAPABILITY_RULES + from synapt.extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.prompt import CAPABILITY_RULES for cap in CAPABILITY_RULES: assert cap in EXTRACTION_CAPABILITIES, f"rule key {cap} not a valid capability" @@ -521,7 +542,7 @@ def test_full_profile_has_no_duplicates(self): assert len(caps) == len(set(caps)), "full profile has duplicate capabilities" def test_build_prompt_succeeds_for_every_capability(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.schema import EXTRACTION_CAPABILITIES modifier_only = {"assertion_signals", "evidence_anchoring"} for cap in EXTRACTION_CAPABILITIES: caps = ["entities", cap] if cap in modifier_only else [cap] diff --git a/tests/python/test_validate.py b/tests/python/test_validate.py index 0d18aa9..042ba53 100644 --- a/tests/python/test_validate.py +++ b/tests/python/test_validate.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "python" / "src")) -from synapt_extract.validate import validate_extraction +from synapt.extract.validate import validate_extraction def _minimal_extraction(**overrides): @@ -279,7 +279,7 @@ def test_unknown_capability(self): assert any("psychic_powers" in e.message for e in result.errors) def test_all_valid_capabilities(self): - from synapt_extract.schema import EXTRACTION_CAPABILITIES + from synapt.extract.schema import EXTRACTION_CAPABILITIES doc = _minimal_extraction(capabilities=sorted(EXTRACTION_CAPABILITIES)) result = validate_extraction(doc) assert result.valid @@ -333,6 +333,85 @@ def test_invalid_temporal_type(self): assert any("type" in e.path for e in result.errors) +class TestTemporalRefRole: + """The validity ROLE (direction) enrichment — config/design/extract-temporal-role- + 2026-07-14.md. role is BASE-tier (no temporal_classes capability needed), optional + (existing {raw, resolved} consumers still validate), and enum-constrained.""" + + def test_public_typeddict_declares_role(self): + """GUARD (Sentinel's finding): the exported public SynaptTemporalRef TypedDict must + declare `role`. The v0.6 runtime EMITS role (builder schema + validation accept it), + so a public type that omitted it would reject its own runtime output — the exact + type/runtime mismatch this guards against. Also confirms the declared enum members + exactly match the validator's accepted set (single source of truth for the role + vocabulary across the type + the validator).""" + import typing + from synapt.extract.schema import SynaptTemporalRef + from synapt.extract.validate import VALID_TEMPORAL_ROLES + + assert "role" in SynaptTemporalRef.__annotations__ + # schema.py uses `from __future__ import annotations`, so raw __annotations__ are + # ForwardRef strings — resolve with get_type_hints before reading the Literal members. + role_type = typing.get_type_hints(SynaptTemporalRef)["role"] + assert set(typing.get_args(role_type)) == set(VALID_TEMPORAL_ROLES) + + @pytest.mark.parametrize("role", ["effective", "expiry", "range", "superseded", "point"]) + def test_valid_role_values(self, role): + temporal_ref = {"version": "1", "raw": "some date reference", "role": role, "resolved": "2026-04-28"} + if role == "range": + temporal_ref["resolved_end"] = "2026-05-01" + doc = _minimal_extraction(temporal_refs=[temporal_ref]) + result = validate_extraction(doc) + assert result.valid, result.errors + + def test_role_absent_still_valid(self): + """Backward compat: a temporal ref with NO role (existing/older extractions, or a + caller not requesting role classification) still validates.""" + doc = _minimal_extraction(temporal_refs=[{ + "version": "1", "raw": "next Tuesday", "resolved": "2026-04-28", + }]) + result = validate_extraction(doc) + assert result.valid + + def test_invalid_role_value_rejected(self): + doc = _minimal_extraction(temporal_refs=[{ + "version": "1", "raw": "sometime", "role": "urgent", "resolved": "2026-04-28", + }]) + result = validate_extraction(doc) + assert not result.valid + assert any("role" in e.path for e in result.errors) + + def test_role_range_without_resolved_end_rejected(self): + """role=="range" requires resolved_end, mirroring the existing type=="range" rule + (TestTemporalRangeConstraints) — a SEPARATE check, since role can appear without type + now that role is base-tier and type stays temporal_classes-gated.""" + doc = _minimal_extraction(temporal_refs=[{ + "version": "1", "raw": "April 20 to May 1", "role": "range", "resolved": "2026-04-20", + }]) + result = validate_extraction(doc) + assert not result.valid + assert any("resolved_end" in e.path or "resolved_end" in e.message for e in result.errors) + + def test_role_present_without_temporal_classes_capability_still_valid(self): + """The load-bearing base-tier requirement: role must validate WITHOUT temporal_classes + being present anywhere in the document — it is not gated behind that capability.""" + doc = _minimal_extraction( + temporal_refs=[{"version": "1", "raw": "expires soon", "role": "expiry", "resolved": "2026-04-30"}], + capabilities=["entities", "goals", "themes", "temporal_refs"], + ) + result = validate_extraction(doc) + assert result.valid, result.errors + + def test_role_effective_without_type_valid(self): + """role can appear on its own, without the temporal_classes-gated `type` field at + all — confirms role and type are independent, not a hidden pairing requirement.""" + doc = _minimal_extraction(temporal_refs=[{ + "version": "1", "raw": "effective March 2026", "role": "effective", "resolved": "2026-03-01", + }]) + result = validate_extraction(doc) + assert result.valid, result.errors + + class TestProducedByFormat: def test_produced_by_requires_scheme(self):