Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ jobs:
- run: pip install build

- run: |
rm -r src/synapt_extract/prompts 2>/dev/null || true
test -d ../../prompts && cp -r ../../prompts src/synapt_extract/prompts || true
rm -r src/synapt/extract/prompts 2>/dev/null || true
test -d ../../prompts && cp -r ../../prompts src/synapt/extract/prompts || true

- run: python -m build

Expand Down Expand Up @@ -154,7 +154,7 @@ jobs:

- name: Verify Python package schemas match root schemas
run: |
diff -r schemas packages/python/src/synapt_extract/schemas
diff -r schemas packages/python/src/synapt/extract/schemas
echo "✅ Python package schemas match root schemas"

schema-url-check:
Expand Down Expand Up @@ -205,8 +205,8 @@ jobs:
SOURCE_DATE_EPOCH: "1704067200"
run: |
pip install build
rm -r src/synapt_extract/prompts 2>/dev/null || true
test -d ../../prompts && cp -r ../../prompts src/synapt_extract/prompts || true
rm -r src/synapt/extract/prompts 2>/dev/null || true
test -d ../../prompts && cp -r ../../prompts src/synapt/extract/prompts || true
python -m build --outdir /tmp/py-dist-1
rm -r src/synapt_extract.egg-info build 2>/dev/null || true
python -m build --outdir /tmp/py-dist-2
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/publish-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ jobs:
- run: pip install build

- run: |
rm -r src/synapt_extract/prompts 2>/dev/null || true
test -d ../../prompts && cp -r ../../prompts src/synapt_extract/prompts || true
rm -r src/synapt/extract/prompts 2>/dev/null || true
test -d ../../prompts && cp -r ../../prompts src/synapt/extract/prompts || true

- run: python -m build

Expand Down
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ dist/
!tests/security-probes/**/*.js

packages/ts/prompts/
packages/python/src/synapt_extract/prompts/
packages/python/src/synapt/extract/prompts/

__pycache__/
*.pyc
Expand Down
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ console.log(result.validation); // { valid: true, errors: [] }
### Python

```python
from synapt_extract import (
from synapt.extract import (
build_extraction_prompt,
finalize_extraction,
FinalizeContext,
Expand Down Expand Up @@ -124,7 +124,7 @@ const built = builder.build({ name: "synapt_extract_stage1" });
### Python

```python
from synapt_extract import create_extraction_builder
from synapt.extract import create_extraction_builder

builder = (
create_extraction_builder(text, profile="standard")
Expand Down Expand Up @@ -173,7 +173,7 @@ const result = await extractOpenAI(text, new OpenAI(), {

```python
from openai import OpenAI
from synapt_extract import create_extraction_builder, extract_openai
from synapt.extract import create_extraction_builder, extract_openai

builder = (
create_extraction_builder(text)
Expand All @@ -192,7 +192,7 @@ result = await extract_openai(
)
```

The returned result includes `artifactBundle` / `artifact_bundle`. TypeScript exports the Node artifact writer at `@synapt-dev/extract/artifacts`; Python exports `write_artifact_bundle()` from `synapt_extract`.
The returned result includes `artifactBundle` / `artifact_bundle`. TypeScript exports the Node artifact writer at `@synapt-dev/extract/artifacts`; Python exports `write_artifact_bundle()` from `synapt.extract`.

```typescript
import { createExtractionBuilder, extract } from "@synapt-dev/extract";
Expand Down Expand Up @@ -227,7 +227,7 @@ const result = await extract(text, {
```

```python
from synapt_extract import create_extraction_builder, extract
from synapt.extract import create_extraction_builder, extract

builder = (
create_extraction_builder(text)
Expand Down
4 changes: 2 additions & 2 deletions SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ sha256sum *.tgz

# Python (wheel)
cd packages/python
cp -r ../../prompts src/synapt_extract/prompts
cp -r ../../schemas src/synapt_extract/schemas
cp -r ../../prompts src/synapt/extract/prompts
cp -r ../../schemas src/synapt/extract/schemas
SOURCE_DATE_EPOCH=1704067200 python -m build
sha256sum dist/*.whl
```
Expand Down
6 changes: 3 additions & 3 deletions examples/dogfood.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages" / "python" / "src"))

import anthropic
from synapt_extract.prompt import build_extraction_prompt
from synapt_extract.finalize import FinalizeContext, finalize_extraction
from synapt_extract.validate import validate_extraction
from synapt.extract.prompt import build_extraction_prompt
from synapt.extract.finalize import FinalizeContext, finalize_extraction
from synapt.extract.validate import validate_extraction

CONVERSATION = """\
Session: Weekly check-in with Marcus, April 22, 2026
Expand Down
8 changes: 4 additions & 4 deletions packages/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ pip install synapt-extract
## Quick start

```python
from synapt_extract import (
from synapt.extract import (
build_extraction_prompt,
finalize_extraction,
FinalizeContext,
Expand Down Expand Up @@ -50,7 +50,7 @@ assert result.validation.valid
Use the builder when the model API supports structured output. It resolves capabilities once, then builds the matching prompt, Stage 1 JSON schema, OpenAI response format, finalized packet schema, and optional finalization context.

```python
from synapt_extract import create_extraction_builder
from synapt.extract import create_extraction_builder

builder = (
create_extraction_builder(text, profile="standard")
Expand Down Expand Up @@ -81,7 +81,7 @@ For OpenAI-compatible clients, use the thin adapter instead of writing callbacks

```python
from openai import OpenAI
from synapt_extract import create_extraction_builder, extract_openai
from synapt.extract import create_extraction_builder, extract_openai

builder = (
create_extraction_builder(text)
Expand All @@ -103,7 +103,7 @@ result = await extract_openai(
The returned result includes `artifact_bundle`. `write_artifact_bundle()` can also write a bundle created from any `extract()` result.

```python
from synapt_extract import create_extraction_builder, extract
from synapt.extract import create_extraction_builder, extract

builder = (
create_extraction_builder(text)
Expand Down
3 changes: 2 additions & 1 deletion packages/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ Schema = "https://synapt.dev/schemas/extract/v1.json"

[tool.setuptools.packages.find]
where = ["src"]
namespaces = true

[tool.setuptools.package-data]
synapt_extract = ["prompts/**/*.txt", "prompts/**/*.json", "schemas/**/*.json"]
"synapt.extract" = ["prompts/**/*.txt", "prompts/**/*.json", "schemas/**/*.json"]
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""synapt-extract: SynaptExtraction IL v1 schema, validation, and finalization."""

from synapt_extract.schema import (
from synapt.extract.schema import (
SynaptExtraction,
SynaptEntity,
SynaptGoal,
Expand All @@ -16,9 +16,9 @@
SynaptAssertionSignals,
SynaptTemporalRef,
)
from synapt_extract.validate import validate_extraction, ValidationResult, ValidationError
from synapt_extract.finalize import finalize_extraction, FinalizeContext, FinalizeResult
from synapt_extract.prompt import (
from synapt.extract.validate import validate_extraction, ValidationResult, ValidationError
from synapt.extract.finalize import finalize_extraction, FinalizeContext, FinalizeResult
from synapt.extract.prompt import (
build_extraction_prompt,
capability_embedding_input,
profile_capabilities,
Expand All @@ -27,14 +27,14 @@
CAPABILITY_REGISTRY,
STANDARD_EMBEDDING_INPUTS,
)
from synapt_extract.builder import (
from synapt.extract.builder import (
ExtractionBuilder,
build_finalized_extraction_schema,
build_extraction_schema,
build_extraction_response_format,
create_extraction_builder,
)
from synapt_extract.extract import (
from synapt.extract.extract import (
extract,
normalize_llm_response,
run_extraction,
Expand All @@ -52,15 +52,22 @@
NormalizedLlmResponse,
UsageSummary,
)
from synapt_extract.artifacts import (
from synapt.extract.artifacts import (
create_artifact_bundle,
sha256_text,
write_artifact_bundle,
)
from synapt_extract.openai import (
from synapt.extract.openai import (
extract_openai,
OpenAIExtractResult,
)
from synapt.extract.batch import (
BatchFailureReason,
BatchInferRequest,
BatchUnit,
BatchUnitResult,
extract_batch,
)

__all__ = [
"SynaptExtraction",
Expand Down Expand Up @@ -116,4 +123,9 @@
"write_artifact_bundle",
"extract_openai",
"OpenAIExtractResult",
"BatchFailureReason",
"BatchInferRequest",
"BatchUnit",
"BatchUnitResult",
"extract_batch",
]
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from pathlib import Path
from typing import Any

from synapt_extract.extract import ExtractResult
from synapt.extract.extract import ExtractResult


JsonObject = dict[str, Any]
Expand Down
130 changes: 130 additions & 0 deletions packages/python/src/synapt/extract/batch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
"""Batch Stage-1 extraction primitive for SynaptExtraction.

SKELETON (recall#868 → extract_batch). API conformed to the pinned contract
(config/design/extract-batch-limits-characterization-2026-07-13.md §"Contract
decisions") AND to Sentinel's spec (extract#28, tests/python/test_extract_batch.py).
Every body raises NotImplementedError — the implementation lands in the follow-up
impl PR (TDD: this skeleton makes the spec COLLECT and run RED, not ImportError).

Why this primitive exists
-------------------------
Atlas's characterization found the generic single-text builder cannot reliably
produce a schema-valid packet even for ONE clean pre-identified unit (NO_VIABLE_N
at N=1). The failure is MALFORMATION on GROUNDED content (40/40 source-supported),
not confabulation — the model returns the right facts in the wrong shape. Fixed
with structural machinery (shaping + per-item validation + fallback), not prompt
tuning. A NEW primitive, not a wrapper/loop over the generic builder.

Contract (pinned + spec-confirmed)
----------------------------------
• Input: list[BatchUnit(id, text, capabilities?)] — explicit attribution; the
id rides into the output as source_unit_id (boundaries stay out-of-band, never
in model-visible text).
• Inference: an injected `infer` seam receiving a request {prompt, messages,
capabilities} and returning a completion string. ZERO recall dependency.
• v1 strategy: PER-UNIT (one infer call per unit) — trivially out-of-band, clean
1:1 attribution. batch-all / safe-N are future INTERNAL ladder rungs, not v1
contract surface. Retry: one deterministic retry per failed unit (2 attempts
total), then a terminal marker from the last failure class; never replay a
successful neighbor.
• Output: COUNT-INVARIANT len(out)==len(in). Each unit → an "ok" BatchUnitResult
(valid envelope) OR a terminal {source_unit_id, status:"failed", reason} marker.
reason ∈ BatchFailureReason. No silent drops.
• Shaping (folded from recall #870/#871, held/superseded):
Class-A PRE-parse text hygiene — strip ``` fences + `//` comments, STRING-
LITERAL-AWARE (a `//` inside a JSON string, e.g. a URL, must survive).
Class-B POST-parse coercion — capability set is the arbiter: in-scope fields
coerced (scalar→array, decided_at null→omit, category→valid/default),
out-of-scope dropped; temporal_refs → schema-valid raw/resolved only.

Harvest map: scratchpad/extract_batch_craft_harvest.md. Boundary: OSS.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Any, Callable, Literal, TypedDict

from synapt.extract.finalize import finalize_extraction

# Terminal per-unit failure reasons (Q5). A Literal (not an Enum) so the spec's
# get_args(BatchFailureReason) reads the members. "merged" is reserved for a future
# batch-all path; the per-unit v1 path never emits it.
BatchFailureReason = Literal["unparseable", "schema_invalid", "dropped", "merged"]


class BatchInferRequest(TypedDict):
"""The exact request the injected `infer` seam receives (Q-D). No unit id /
boundary tag ever appears here — boundaries stay in extract_batch bookkeeping,
out of model-visible text."""

prompt: str
messages: list[dict[str, str]]
capabilities: list[str]


# The injected inference seam (Q4): request → completion. The caller (recall) passes
# a model-backed callable; tests pass a deterministic/recorded one. Zero recall dep.
Inferer = Callable[[BatchInferRequest], str]


@dataclass
class BatchUnit:
"""One pre-identified unit to extract (Q1). ``id`` is stable and rides into the
output as ``source_unit_id`` so merge/split/drop is detectable. ``capabilities``
optionally overrides the per-call default for this unit."""

id: str
text: str
capabilities: list[str] | None = None


@dataclass
class BatchUnitResult:
"""Per-unit outcome (Q5). ``status`` "ok" sets ``extraction``; "failed" sets
``reason``. ``source_unit_id`` ties the slot back to its BatchUnit."""

source_unit_id: str
status: str # "ok" | "failed"
extraction: Any | None = None # a finalized SynaptExtraction, or None
reason: BatchFailureReason | None = None


async def extract_batch(
units: list[BatchUnit],
*,
infer: Inferer,
produced_by: str,
capabilities: list[str] | None = None,
) -> list[BatchUnitResult]:
"""Shape + validate a batch of pre-identified units into per-unit envelopes.

COUNT-INVARIANT: returns exactly one BatchUnitResult per input unit, in a 1:1
slot mapping (Q5). extract_batch owns the reliability orchestration (v1 =
per-unit calls with one deterministic retry per failed unit) driven through the
injected ``infer`` seam, with zero dependency on any specific model client (Q4).
``capabilities`` defaults to the standard profile when omitted (Q3).

SKELETON — body is NotImplementedError; the impl lands in the follow-up PR.
"""
raise NotImplementedError(
"extract_batch skeleton conforms to the pinned contract + spec; the "
"implementation lands in the impl PR (recall#868)."
)


# --- Intended internal decomposition (stubs; bodies in the impl PR) ------------

def _strip_output_hygiene(raw: str) -> str:
"""Class-A PRE-parse (NET-NEW): strip ``` fences + ``//`` comments so grounded-
but-wrapped JSON parses. STRING-LITERAL-AWARE — a ``//`` inside a JSON string
value (e.g. ``https://…``) is preserved; only real line-comments are removed."""
raise NotImplementedError


def _coerce_shape(parsed: dict, capabilities: list[str]) -> dict:
"""Class-B POST-parse (harvest ``_sanitize_stage1_output`` whitelist backbone):
the capability set is the arbiter (Q2) — in-scope fields coerced (scalar→array,
``decided_at`` null→omit, ``category``→valid/default), out-of-scope dropped;
``temporal_refs`` coerced to schema-valid ``raw``/``resolved`` only."""
raise NotImplementedError
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
from dataclasses import dataclass
from typing import Any

from synapt_extract.finalize import FinalizeContext, FinalizeResult, finalize_extraction
from synapt_extract.prompt import (
from synapt.extract.finalize import FinalizeContext, FinalizeResult, finalize_extraction
from synapt.extract.prompt import (
CANONICAL_ORDER,
STANDARD_EMBEDDING_INPUTS,
build_extraction_prompt,
Expand Down
Loading
Loading