From f436cdbb9bd83dc4147aedea13754292dd9834fe Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 00:46:57 +0700 Subject: [PATCH 1/5] spec: bootstrap Trace conformance baseline --- conformance/README.md | 44 + conformance/audit.md | 59 ++ conformance/diagnostic-registry.json | 149 ++++ conformance/diagnostic-registry.schema.json | 26 + conformance/manifest.json | 22 + conformance/manifest.schema.json | 77 ++ conformance/requirements.json | 792 ++++++++++++++++++ conformance/requirements.schema.json | 94 +++ ...026-08-03-conformance-baseline-proposal.md | 213 +++++ decisions/README.md | 1 + examples/prompt-trace-0.3-draft.example.json | 87 ++ schemas/prompt-trace-0.3-draft.schema.json | 643 ++++++++++++++ 12 files changed, 2207 insertions(+) create mode 100644 conformance/README.md create mode 100644 conformance/audit.md create mode 100644 conformance/diagnostic-registry.json create mode 100644 conformance/diagnostic-registry.schema.json create mode 100644 conformance/manifest.json create mode 100644 conformance/manifest.schema.json create mode 100644 conformance/requirements.json create mode 100644 conformance/requirements.schema.json create mode 100644 decisions/2026-08-03-conformance-baseline-proposal.md create mode 100644 examples/prompt-trace-0.3-draft.example.json create mode 100644 schemas/prompt-trace-0.3-draft.schema.json diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..9c97d53 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,44 @@ +# PromptSyntax conformance corpus + +This directory will contain the canonical, language-independent PromptSyntax conformance +corpus. The specification and expected vectors are the oracle. Implementations consume the +corpus but do not define its expected outcomes. + +## Current status + +The corpus is being bootstrapped against the specification draft at commit +`7456634817f4eb68c8909a58b476505e6fd7c063`. It is not yet a released conformance suite and +must not be used for an unqualified conformance claim. + +Normative vectors are blocked until the v0.3 baseline reconciles the specification and Trace +schema. The initial machine-readable requirement inventory may contain blocked or proposed +entries so that gaps remain visible rather than being silently omitted. + +## Planned ownership + +- `promptsyntax.org` owns specifications, schemas, requirements, vectors, and published + reports. +- `PromptSyntax-rs` owns the stable-Rust validator and conformance runner. +- Each implementation exposes a thin adapter that contains no expected answers. + +The normative runner does not require Python, Node, Bun, a provider SDK, a network +connection, or model credentials. The TypeScript implementation may be invoked separately +with Bun when producing cross-language evidence. + +## Conformance targets + +1. **Corpus:** metadata, references, artifacts, hashes, and coverage are internally valid. +2. **Core parser:** source produces exact lossless segments, projections, spans, directives, + and diagnostics. +3. **Trace document:** a trace and declared artifacts satisfy structural and semantic rules. +4. **Trace producer:** deterministic execution facts produce a trace consistent with those + facts. + +Passing document validation does not prove that a deployed venue reported execution facts +truthfully. Independent collection or attestation is a separate trust mechanism. + +## Development rule + +If the specification does not determine an expected result, do not add a normative vector. +Record the ambiguity, resolve it in the specification and decision log, and only then freeze +the expected outcome. diff --git a/conformance/audit.md b/conformance/audit.md new file mode 100644 index 0000000..2066fe3 --- /dev/null +++ b/conformance/audit.md @@ -0,0 +1,59 @@ +# Conformance baseline audit + +Audit date: 2026-08-03 + +Audited baseline: `7456634817f4eb68c8909a58b476505e6fd7c063` + +This audit records issues that prevent the current working draft from serving as a +deterministic conformance oracle. It is not itself normative. + +## Release blockers + +| ID | Finding | Why it blocks a test oracle | Proposed disposition | +|---|---|---|---| +| A01 | The document declares v0.2 while containing settled v0.2.1 material and a proposed v0.2.2 section. | A case cannot cite one coherent behavior version. | Consolidate settled material as v0.3 and exclude proposed §13.2 from the first profile. | +| A02 | Entity fill entries cannot distinguish kept, explicit fallback, venue substitution, and refusal. | The paper's central requested-versus-applied outcome has no unambiguous machine representation. | Use a discriminated entity outcome with `kept`, `fallback`, `substituted`, and `refused`. | +| A03 | §10.5 requires `SAFETY_BLOCKED`, but §8.2 and the schema omit it. | The same document is conformant in prose and impossible in schema. | Reconcile an outcome reason registry and add missing availability, substitution, and safety reasons. | +| A04 | User-initiated `segments`, `steps`, and `resolution` are normative but optional in the schema. | A schema-valid trace can omit the compiled prompt and resolution evidence. | Add structural conditionals where practical and semantic checks for the remainder. | +| A05 | `routing.bound` is always required. | A strict request refused before invocation has no legal representation. | Use a routing outcome union or conditionally require `bound` only when an invocation occurred. | +| A06 | R1 names byte-exact reproduction without defining independent request bytes or complete external and withheld resolution semantics. | A trace can only agree with itself, not prove what was sent. | Define UTF-8 byte semantics and a producer transcript containing actual request bytes. | +| A07 | The schema permits arbitrary unknown fields and does not define an extension policy. | Independent validators may accept different dialects while claiming the same version. | Reject unknown fields by default or isolate them in a namespaced extension object. | +| A08 | Hash strings, lengths, IDs, indices, and graph relations are weakly constrained. | Malformed evidence can pass structural validation. | Define exact hash syntax, non-negative lengths, uniqueness, ordering, pointer scope, and graph rules. | +| A09 | §10.5 requires typed refusal reason, authority, trigger provenance, and recourse, while the schema only partially represents them. | Safety, policy, authorization, and availability refusals cannot be tested consistently. | Add a discriminated refusal object and define required fields by reason. | +| A10 | Oversight prose requires sequence gaps, retention, and residency metadata that the schema cannot represent. | Oversight conformance cannot be claimed from the published artifact. | Exclude oversight from the first headline profile and reconcile its schema before claiming it. | +| A11 | Boundary events always require received content. | Failed or cancelled tool calls cannot be represented honestly. | Add success and failure outcome variants without erasing what crossed the boundary. | +| A12 | The generic fill-entry status vocabulary mixes entities, parameters, budgets, and actions. | Legal status and code combinations are ambiguous. | Use discriminated fill-entry variants with type-specific status vocabularies. | +| A13 | Proposed §13.2 contains normative words but lacks capability and Trace schemas. | Including it would create untestable Core requirements. | Keep it candidate-only until two implementations and machine-testable shapes exist. | +| A14 | The example uses short strings as SHA-256 values, which the schema accepts. | It normalizes evidence that is not a valid SHA-256 digest. | Require 64 lowercase hexadecimal characters and update examples with fictional fixtures. | +| A15 | The current Core fixture is manually duplicated in Rust and TypeScript repositories. | Drift can occur without detection and the specification does not own the oracle. | Promote it into this directory and have both adapters consume one corpus. | +| A16 | No current CI validates the schema against the example or checks semantic Trace rules. | Published artifacts can drift independently. | Add Rust corpus, schema, semantic, and producer checks in separate CI lanes. | + +## Non-blocking but required before stronger profiles + +| ID | Finding | Required follow-up | +|---|---|---| +| A17 | The transparency floor is explicitly open. | Freeze placeholder contents before claiming full tier-relative completeness. | +| A18 | Integrity chaining lacks canonical serialization and verification procedures. | Define these in an oversight profile before making integrity a `MUST`. | +| A19 | Capability document shape is illustrative, not schema-backed. | Publish a versioned capability schema before full PS/Capabilities conformance. | +| A20 | Budget measurement profiles remain open. | Keep deterministic synthetic measurements in core producer tests and defer vendor accounting claims. | +| A21 | Default namespace search order remains open. | Core parser vectors may test declared environments, but resolver conformance waits for a settled order. | +| A22 | Nested-span conflict semantics remain open. | Exclude ambiguous conflict cases from normative Core vectors until resolved. | +| A23 | Localized keyword governance remains open. | Test canonical English keywords and settled Unicode behavior; keep localized aliases optional. | + +## Safe work that can proceed before v0.3 is frozen + +- Define the corpus metadata and requirement schemas. +- Inventory every settled, proposed, and open requirement. +- Promote existing Core fixtures with draft provenance and stable candidate IDs. +- Implement a Rust corpus self-checker against draft data. +- Implement adapter plumbing without expected semantic answers. +- Draft minimal valid and invalid Trace mutations, marked non-normative until their rules are + frozen. + +## Work that must wait for the oracle + +- Publishing a normative suite version. +- Calling any implementation conformant to PS/Trace v0.3. +- Freezing entity substitution and refusal vectors. +- Freezing R1 vectors involving external or withheld content. +- Claiming oversight profile coverage. diff --git a/conformance/diagnostic-registry.json b/conformance/diagnostic-registry.json new file mode 100644 index 0000000..ff33ab6 --- /dev/null +++ b/conformance/diagnostic-registry.json @@ -0,0 +1,149 @@ +{ + "format_version": "0.1-draft", + "diagnostics": [ + { + "code": "CORPUS_BLOCKED_WITHOUT_REASON", + "phase": "corpus", + "severity": "error", + "description": "A blocked requirement does not identify an audit blocker." + }, + { + "code": "CORPUS_BLOCKER_DUPLICATE", + "phase": "corpus", + "severity": "error", + "description": "A requirement lists the same audit blocker more than once." + }, + { + "code": "CORPUS_BLOCKER_ID_INVALID", + "phase": "corpus", + "severity": "error", + "description": "An audit blocker identifier does not use the required A01 form." + }, + { + "code": "CORPUS_BLOCKER_ON_UNBLOCKED_REQUIREMENT", + "phase": "corpus", + "severity": "error", + "description": "A requirement declares blockers while its testability is not blocked." + }, + { + "code": "CORPUS_FORMAT_UNSUPPORTED", + "phase": "corpus", + "severity": "error", + "description": "The corpus format version is not supported by this runner." + }, + { + "code": "CORPUS_JSON_INVALID", + "phase": "corpus", + "severity": "error", + "description": "The corpus document is not valid JSON or does not deserialize to the closed typed model." + }, + { + "code": "CORPUS_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "The corpus file could not be read." + }, + { + "code": "CORPUS_REQUIREMENTS_EMPTY", + "phase": "corpus", + "severity": "error", + "description": "The requirement inventory contains no requirements." + }, + { + "code": "CORPUS_REQUIREMENT_ID_DUPLICATE", + "phase": "corpus", + "severity": "error", + "description": "A requirement identifier occurs more than once." + }, + { + "code": "CORPUS_REQUIREMENT_ID_INVALID", + "phase": "corpus", + "severity": "error", + "description": "A requirement identifier is malformed or disagrees with its declared layer." + }, + { + "code": "CORPUS_REQUIREMENT_STATEMENT_EMPTY", + "phase": "corpus", + "severity": "error", + "description": "A requirement has no non-whitespace statement." + }, + { + "code": "CORPUS_SOURCE_ANCHOR_INVALID", + "phase": "corpus", + "severity": "error", + "description": "A candidate specification anchor is not a safe lowercase identifier." + }, + { + "code": "CORPUS_SOURCE_PATH_INVALID", + "phase": "corpus", + "severity": "error", + "description": "A source path is absolute, empty, or contains parent traversal." + }, + { + "code": "CORPUS_SOURCE_SECTION_EMPTY", + "phase": "corpus", + "severity": "error", + "description": "A requirement does not identify a source section." + }, + { + "code": "CORPUS_SPEC_COMMIT_INVALID", + "phase": "corpus", + "severity": "error", + "description": "The pinned specification commit is not 40 lowercase hexadecimal characters." + }, + { + "code": "CORPUS_TARGETS_EMPTY", + "phase": "corpus", + "severity": "error", + "description": "A requirement does not identify a conformance target." + }, + { + "code": "CORPUS_TARGET_DUPLICATE", + "phase": "corpus", + "severity": "error", + "description": "A requirement lists the same conformance target more than once." + }, + { + "code": "INSTANCE_JSON_INVALID", + "phase": "instance", + "severity": "error", + "description": "The instance is not valid JSON." + }, + { + "code": "INSTANCE_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "The instance file could not be read." + }, + { + "code": "SCHEMA_BUILD_FAILED", + "phase": "schema", + "severity": "error", + "description": "A Draft 2020-12 validator could not be built from the schema." + }, + { + "code": "SCHEMA_INSTANCE_INVALID", + "phase": "instance", + "severity": "error", + "description": "An instance violates its declared schema." + }, + { + "code": "SCHEMA_JSON_INVALID", + "phase": "schema", + "severity": "error", + "description": "The schema is not valid JSON." + }, + { + "code": "SCHEMA_META_INVALID", + "phase": "schema", + "severity": "error", + "description": "The schema violates its JSON Schema meta-schema." + }, + { + "code": "SCHEMA_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "The schema file could not be read." + } + ] +} diff --git a/conformance/diagnostic-registry.schema.json b/conformance/diagnostic-registry.schema.json new file mode 100644 index 0000000..9fe53b0 --- /dev/null +++ b/conformance/diagnostic-registry.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/diagnostics/0.1-draft", + "title": "PromptSyntax conformance diagnostic registry", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "diagnostics"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "diagnostics": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "phase", "severity", "description"], + "properties": { + "code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" }, + "phase": { "enum": ["io", "corpus", "schema", "instance", "semantic", "producer"] }, + "severity": { "enum": ["error", "warning"] }, + "description": { "type": "string", "minLength": 1 } + } + } + } + } +} diff --git a/conformance/manifest.json b/conformance/manifest.json new file mode 100644 index 0000000..984b937 --- /dev/null +++ b/conformance/manifest.json @@ -0,0 +1,22 @@ +{ + "format_version": "0.1-draft", + "suite": { + "id": "promptsyntax-conformance", + "version": "0.0.0-bootstrap", + "status": "bootstrap" + }, + "spec": { + "path": "spec/SPEC.md", + "commit": "7456634817f4eb68c8909a58b476505e6fd7c063", + "declared_version": "0.2 with settled 0.2.1 and proposed 0.2.2 material", + "baseline_status": "working-draft" + }, + "files": { + "requirements": "conformance/requirements.json", + "requirements_schema": "conformance/requirements.schema.json", + "diagnostics": "conformance/diagnostic-registry.json", + "diagnostics_schema": "conformance/diagnostic-registry.schema.json" + }, + "families": [], + "claims": [] +} diff --git a/conformance/manifest.schema.json b/conformance/manifest.schema.json new file mode 100644 index 0000000..0262286 --- /dev/null +++ b/conformance/manifest.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/manifest/0.1-draft", + "title": "PromptSyntax conformance suite manifest", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "suite", "spec", "files", "families", "claims"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "suite": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "status"], + "properties": { + "id": { "const": "promptsyntax-conformance" }, + "version": { "type": "string", "minLength": 1 }, + "status": { "enum": ["bootstrap", "candidate", "released"] } + } + }, + "spec": { + "type": "object", + "additionalProperties": false, + "required": ["path", "commit", "declared_version", "baseline_status"], + "properties": { + "path": { "type": "string", "minLength": 1 }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "declared_version": { "type": "string", "minLength": 1 }, + "baseline_status": { "enum": ["working-draft", "frozen"] } + } + }, + "files": { + "type": "object", + "additionalProperties": false, + "required": ["requirements", "requirements_schema", "diagnostics", "diagnostics_schema"], + "properties": { + "requirements": { "type": "string", "minLength": 1 }, + "requirements_schema": { "type": "string", "minLength": 1 }, + "diagnostics": { "type": "string", "minLength": 1 }, + "diagnostics_schema": { "type": "string", "minLength": 1 } + } + }, + "families": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "layer", "target", "version", "path", "status"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, + "layer": { "enum": ["core", "capabilities", "execution", "trace"] }, + "target": { "enum": ["core-parser", "trace-document", "trace-producer"] }, + "version": { "type": "string", "minLength": 1 }, + "path": { "type": "string", "minLength": 1 }, + "status": { "enum": ["draft", "normative"] } + } + } + }, + "claims": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["profile", "families", "status"], + "properties": { + "profile": { "type": "string", "minLength": 1 }, + "families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string" } + }, + "status": { "enum": ["candidate", "released"] } + } + } + } + } +} diff --git a/conformance/requirements.json b/conformance/requirements.json new file mode 100644 index 0000000..e51548f --- /dev/null +++ b/conformance/requirements.json @@ -0,0 +1,792 @@ +{ + "format_version": "0.1-draft", + "spec": { + "path": "spec/SPEC.md", + "commit": "7456634817f4eb68c8909a58b476505e6fd7c063", + "declared_version": "0.2 with settled 0.2.1 and proposed 0.2.2 material", + "baseline_status": "working-draft" + }, + "requirements": [ + { + "id": "PS-META-CLAIM-001", + "layer": "meta", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "4", "candidate_anchor": "ps-meta-claim-001" }, + "statement": "A conformance claim names every claimed PromptSyntax layer and version.", + "testability": "manual", + "targets": ["documentation"] + }, + { + "id": "PS-META-CLAIM-002", + "layer": "meta", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "4", "candidate_anchor": "ps-meta-claim-002" }, + "statement": "An implementation described only as PS-shaped must not be presented as conformant.", + "testability": "manual", + "targets": ["documentation"] + }, + { + "id": "PS-META-DET-001", + "layer": "meta", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "14", "candidate_anchor": "ps-meta-det-001" }, + "statement": "Documentation and conformance claims distinguish routing-contract determinism, generation determinism, and semantic reproducibility.", + "testability": "manual", + "targets": ["documentation"] + }, + + { + "id": "PS-CORE-ZERO-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "3", "candidate_anchor": "ps-core-zero-001" }, + "statement": "A prompt containing no directives is valid PromptSyntax and remains ordinary natural-language content.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-SPAN-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "5.2", "candidate_anchor": "ps-core-span-001" }, + "statement": "A model-routing reference inside a span is rejected rather than ignored and reports SPAN_EXECUTION_UNSUPPORTED.", + "testability": "automatic", + "targets": ["core-parser", "execution-resolver"] + }, + { + "id": "PS-CORE-BIND-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "6.2", "candidate_anchor": "ps-core-bind-001" }, + "statement": "Resolution binds each reference to a globally canonical, immutable, versioned identifier.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document"] + }, + { + "id": "PS-CORE-BIND-002", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "6.2", "candidate_anchor": "ps-core-bind-002" }, + "statement": "A trace never reports an unpinned entity binding.", + "testability": "automatic", + "targets": ["trace-document"] + }, + { + "id": "PS-CORE-AMBIG-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "6.3", "candidate_anchor": "ps-core-ambig-001" }, + "statement": "Multiple matching entities produce a surfaced ENTITY_AMBIGUOUS result.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-CORE-AMBIG-002", + "layer": "core", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "6.3", "candidate_anchor": "ps-core-ambig-002" }, + "statement": "An implementation must not silently choose among multiple matching entities.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-CORE-ACTION-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "6.4", "candidate_anchor": "ps-core-action-001" }, + "statement": "A consequential action is fully qualified or explicitly confirmed by the user.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-CORE-STRICT-MODE-001", + "layer": "core", + "normative_status": "settled", + "strength": "should", + "source": { "path": "spec/SPEC.md", "section": "6.4", "candidate_anchor": "ps-core-strict-mode-001" }, + "statement": "A venue should offer a mode in which only ps tags and strict frontmatter are executable.", + "testability": "profile", + "targets": ["capability-document", "core-parser"] + }, + { + "id": "PS-CORE-ROUNDTRIP-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "11", "candidate_anchor": "ps-core-roundtrip-001" }, + "statement": "Compiling casual form is equivalent to compiling the strict form produced from that casual compilation.", + "testability": "automatic", + "targets": ["core-parser", "execution-resolver"] + }, + { + "id": "PS-CORE-REIFY-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "11", "candidate_anchor": "ps-core-reify-001" }, + "statement": "Intent reification emits an editable and re-submittable strict form.", + "testability": "automatic", + "targets": ["execution-resolver"] + }, + { + "id": "PS-CORE-PROV-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-prov-001" }, + "statement": "Only explicitly designated authoring segments are parsed for directives.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-INERT-001", + "layer": "core", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-inert-001" }, + "statement": "Retrieved documents, file contents, tool results, quoted messages, and model-generated text do not introduce directives without authority-checked promotion.", + "testability": "automatic", + "targets": ["core-parser", "trace-producer"] + }, + { + "id": "PS-CORE-SERIALIZE-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-serialize-001" }, + "statement": "Canonical serialization carries included content by reference or typed data and never splices it into directive text.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-CORE-PROMOTE-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-promote-001" }, + "statement": "Promotion is explicit, authority-checked, and recorded in the trace, and it changes parse eligibility without granting capability.", + "testability": "blocked", + "targets": ["core-parser", "trace-producer"], + "blocked_by": ["A13"], + "notes": "The general invariant is settled, but the reusable surface and trace shape remain proposed." + }, + { + "id": "PS-CORE-TAG-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-tag-001" }, + "statement": "Only the ps tag namespace is live; all other tag-shaped text remains content.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-BOUNDARY-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-boundary-001" }, + "statement": "Point islands require the defined boundary and a qualified or environment-resolvable name; email addresses and paths remain content.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-ESCAPE-001", + "layer": "core", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-escape-001" }, + "statement": "Escaped sigils and fenced code blocks always remain content.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-HIGHLIGHT-001", + "layer": "core", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-highlight-001" }, + "statement": "An interactive implementation visibly marks every recognized directive before submission.", + "testability": "manual", + "targets": ["interactive-ui"] + }, + { + "id": "PS-CORE-WIDTH-001", + "layer": "core", + "normative_status": "proposed", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-width-001" }, + "statement": "Parsers treat full-width at and slash sigils as equivalent to their ASCII forms.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-UNICODE-ID-001", + "layer": "core", + "normative_status": "proposed", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-unicode-id-001" }, + "statement": "Entity identifiers follow UAX 31 with NFC normalization while canonical keywords remain English.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-SEGMENT-001", + "layer": "core", + "normative_status": "proposed", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-segment-001" }, + "statement": "Island boundaries follow UAX 29 and work in scripts without inter-word spaces.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-CORPUS-I18N-001", + "layer": "core", + "normative_status": "proposed", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-corpus-i18n-001" }, + "statement": "The conformance corpus includes cases for scripts without inter-word spacing.", + "testability": "automatic", + "targets": ["corpus"] + }, + { + "id": "PS-CORE-BIDI-001", + "layer": "core", + "normative_status": "proposed", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-bidi-001" }, + "statement": "A parser rejects or neutralizes bidi control characters inside directive islands without leaking a partial directive.", + "testability": "automatic", + "targets": ["core-parser"] + }, + { + "id": "PS-CORE-SURFACE-001", + "layer": "core", + "normative_status": "proposed", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "13.2", "candidate_anchor": "ps-core-surface-001" }, + "statement": "A standing authoring-surface declaration binds identity, lifecycle, origin, frame, action set, capability scope, replay behavior, and trace obligations.", + "testability": "blocked", + "targets": ["capability-document", "core-parser", "trace-producer"], + "blocked_by": ["A13"] + }, + { + "id": "PS-CORE-SURFACE-002", + "layer": "core", + "normative_status": "proposed", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "13.2", "candidate_anchor": "ps-core-surface-002" }, + "statement": "A host must not search undifferentiated transcript, concatenated prose, quoted content, or retrieved data for a surface delimiter.", + "testability": "blocked", + "targets": ["core-parser"], + "blocked_by": ["A13"] + }, + + { + "id": "PS-CAP-DOC-001", + "layer": "capabilities", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "7.1", "candidate_anchor": "ps-cap-doc-001" }, + "statement": "A conformant venue publishes a machine-readable capability document and negotiates against it before fill.", + "testability": "blocked", + "targets": ["capability-document", "execution-resolver"], + "blocked_by": ["A19"] + }, + { + "id": "PS-CAP-VERSION-001", + "layer": "capabilities", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "7.1", "candidate_anchor": "ps-cap-version-001" }, + "statement": "A venue rejects a declared PromptSyntax version it cannot honor instead of degrading silently.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-CAP-PARAM-001", + "layer": "capabilities", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "7.2", "candidate_anchor": "ps-cap-param-001" }, + "statement": "Every parameter records requested, normalized, and applied values with a defined status in the fill report.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-CAP-PARAM-002", + "layer": "capabilities", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "7.2", "candidate_anchor": "ps-cap-param-002" }, + "statement": "An implementation must not claim undocumented parameter passthrough as conforming normalization.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-CAP-TIER-001", + "layer": "capabilities", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.2", "candidate_anchor": "ps-cap-tier-001" }, + "statement": "The capability document declares the trace tiers supported by the venue.", + "testability": "blocked", + "targets": ["capability-document", "trace-document"], + "blocked_by": ["A19"] + }, + + { + "id": "PS-EXEC-STRICT-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-strict-001" }, + "statement": "Strict fulfillment applies every requirement exactly or fails before execution.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-PARTIAL-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-partial-001" }, + "statement": "Partial fulfillment records every dropped requirement.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-BEST-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-best-001" }, + "statement": "Best-effort fulfillment records every degradation.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-DEFAULT-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-default-001" }, + "statement": "The default policy is strict entity binding and partial parameter fulfillment.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-BANG-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-bang-001" }, + "statement": "A bang marker makes the entire directive strict, permitting only declared semantics-preserving translation.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-REASON-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "8.2", "candidate_anchor": "ps-exec-reason-001" }, + "statement": "Every fill failure and disclosed degradation carries a standardized or permitted extension reason code.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A02", "A03", "A12"] + }, + { + "id": "PS-EXEC-BUDGET-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-001" }, + "statement": "A violated budget is a fill failure.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-BUDGET-002", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-002" }, + "statement": "Required cancellation against a venue declaring cancellation unsupported fails before execution with BUDGET_PREFLIGHT_REJECTED.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-BUDGET-003", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-003" }, + "statement": "A timeout that the venue could not cancel reports TIMEOUT_NOT_CANCELLED with observed cost.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-BUDGET-004", + "layer": "execution", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-004" }, + "statement": "An implementation must not silently truncate or otherwise degrade output to satisfy a budget.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-BUDGET-005", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-005" }, + "statement": "A chain-scoped budget applies to the total across fallback attempts.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-FALLBACK-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-001" }, + "statement": "A fill failure is the only trigger for advancing an authored fallback chain.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-FALLBACK-002", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-002" }, + "statement": "The first successful legal fallback step wins.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-FALLBACK-003", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-003" }, + "statement": "The fill report records every fallback attempt in order with outcome, codes, measured cost, and measured latency.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-FALLBACK-004", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-004" }, + "statement": "Every fallback step is checked against the same applicable capability, authorization, consent, data, residency, retention, and connector constraints.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-AUTH-001", + "layer": "execution", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "10.1", "candidate_anchor": "ps-exec-auth-001" }, + "statement": "A lower authority must not expand permissions, budgets, or entity sets granted by higher authority.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-SPECIFICITY-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "10.2", "candidate_anchor": "ps-exec-specificity-001" }, + "statement": "Within one authority level, span scope overrides point scope and point scope overrides document scope.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-producer"] + }, + { + "id": "PS-EXEC-ENVELOPE-001", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "10.3", "candidate_anchor": "ps-exec-envelope-001" }, + "statement": "An out-of-envelope directive fails with ENTITY_UNAUTHORIZED or DATA_POLICY_BLOCKED and is never silently ignored.", + "testability": "automatic", + "targets": ["execution-resolver", "trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-ENVELOPE-002", + "layer": "execution", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "10.3", "candidate_anchor": "ps-exec-envelope-002" }, + "statement": "Every envelope constraint that suppresses a directive is recorded as a typed refusal at the requesting user's tier.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-EXEC-REFUSAL-001", + "layer": "execution", + "normative_status": "proposed", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "10.5", "candidate_anchor": "ps-exec-refusal-001" }, + "statement": "A refusal distinguishes availability, safety, organization or data policy, and authorization, and records authority and available recourse.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A03", "A09"] + }, + + { + "id": "PS-TRACE-R1", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-r1" }, + "statement": "Ordered trace segments reproduce the tier-scoped compiled prompt byte-exactly.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A06"] + }, + { + "id": "PS-TRACE-R2", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-r2" }, + "statement": "Every required segment provenance pointer resolves to a step in the same inference.", + "testability": "automatic", + "targets": ["trace-document"] + }, + { + "id": "PS-TRACE-R3", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-r3" }, + "statement": "Provenance parent chains resolve within their inference and are acyclic.", + "testability": "automatic", + "targets": ["trace-document"] + }, + { + "id": "PS-TRACE-R4", + "layer": "trace", + "normative_status": "settled", + "strength": "must_not", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-r4" }, + "statement": "The user-tier inline threshold must not be lower than 4096 bytes.", + "testability": "automatic", + "targets": ["trace-document"] + }, + { + "id": "PS-TRACE-R5", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-r5" }, + "statement": "An oversight-tier implementation supports a materialized export that resolves required external content inline.", + "testability": "profile", + "targets": ["trace-document", "trace-producer", "oversight-profile"] + }, + { + "id": "PS-TRACE-R6", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-r6" }, + "statement": "Trace coverage is explicitly declared and no unqualified conformance claim is accepted.", + "testability": "automatic", + "targets": ["trace-document", "documentation"] + }, + { + "id": "PS-TRACE-EVENTS-001", + "layer": "trace", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-events-001" }, + "statement": "A Prompt Trace is a turn-level object whose ordered event array interleaves inference and boundary events in occurrence order.", + "testability": "automatic", + "targets": ["trace-document"] + }, + { + "id": "PS-TRACE-ROUTING-001", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-routing-001" }, + "statement": "Every venue invocation records a routing record and fill report.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-TRACE-ASSEMBLY-001", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-assembly-001" }, + "statement": "Every user-initiated inference records full tier-scoped compiled prompt segments.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-TRACE-ASSEMBLY-002", + "layer": "trace", + "normative_status": "settled", + "strength": "should", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-assembly-002" }, + "statement": "An internal continuation should record full tier-scoped compiled prompt segments, and an omission is typed and declared.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-TRACE-BOUNDARY-001", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-boundary-001" }, + "statement": "Every tool input and output crossing the venue boundary is recorded while tool internals remain outside the perimeter.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A11"] + }, + { + "id": "PS-TRACE-CONTENT-001", + "layer": "trace", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-content-001" }, + "statement": "Trace content uses exactly one tier-appropriate inline, external, or withheld state.", + "testability": "automatic", + "targets": ["trace-document"] + }, + { + "id": "PS-TRACE-RESOLUTION-001", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-resolution-001" }, + "statement": "A trace records every binding, deciding rule, surfaced ambiguity, and envelope refusal in its resolution report.", + "testability": "automatic", + "targets": ["trace-document", "trace-producer"] + }, + { + "id": "PS-TRACE-FILL-001", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-fill-001" }, + "statement": "A routing record includes outcome codes, normalization records, and per-attempt outcomes with measured cost and latency.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A02", "A03", "A12"] + }, + { + "id": "PS-TRACE-DRYRUN-001", + "layer": "trace", + "normative_status": "settled", + "strength": "should", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-dryrun-001" }, + "statement": "An implementation should support a pre-execution dry-run trace.", + "testability": "profile", + "targets": ["capability-document", "trace-producer"] + }, + { + "id": "PS-TRACE-SEGMENT-ID-001", + "layer": "trace", + "normative_status": "settled", + "strength": "should", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-segment-id-001" }, + "statement": "An implementation should produce stable segment identifiers suitable for trace diffing.", + "testability": "automatic", + "targets": ["trace-producer"] + }, + { + "id": "PS-TRACE-PRESENTATION-001", + "layer": "trace", + "normative_status": "settled", + "strength": "may", + "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-presentation-001" }, + "statement": "Presentation of conforming trace information is implementation-defined.", + "testability": "manual", + "targets": ["documentation", "interactive-ui"] + }, + { + "id": "PS-TRACE-TIER-USER-001", + "layer": "trace", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.2", "candidate_anchor": "ps-trace-tier-user-001" }, + "statement": "A user-tier trace exposes user-visible sources, selected tools, applied parameters, typed envelope refusals, and the fill report.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A04", "A09"] + }, + { + "id": "PS-TRACE-PLACEHOLDER-001", + "layer": "trace", + "normative_status": "open", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.2", "candidate_anchor": "ps-trace-placeholder-001" }, + "statement": "Content above the current tier appears as a typed and safely sized placeholder.", + "testability": "blocked", + "targets": ["trace-document", "trace-producer"], + "blocked_by": ["A17"] + }, + { + "id": "PS-TRACE-INTEGRITY-001", + "layer": "trace", + "normative_status": "settled", + "strength": "should", + "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-integrity-001" }, + "statement": "Core Trace should support integrity chaining or signing, with stronger requirements defined by oversight profiles.", + "testability": "blocked", + "targets": ["oversight-profile", "trace-document"], + "blocked_by": ["A18"] + }, + { + "id": "PS-TRACE-OVERSIGHT-001", + "layer": "trace", + "normative_status": "settled", + "strength": "must", + "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-oversight-001" }, + "statement": "An implementation claiming oversight-tier support records retention and residency metadata.", + "testability": "blocked", + "targets": ["oversight-profile", "trace-document", "trace-producer"], + "blocked_by": ["A10"] + }, + { + "id": "PS-TRACE-OVERSIGHT-002", + "layer": "trace", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-oversight-002" }, + "statement": "An oversight evidence stream represents omitted events with explicit signed sequence gaps rather than silence.", + "testability": "blocked", + "targets": ["oversight-profile", "trace-document", "trace-producer"], + "blocked_by": ["A10", "A18"] + }, + { + "id": "PS-TRACE-OVERSIGHT-003", + "layer": "trace", + "normative_status": "settled", + "strength": "invariant", + "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-oversight-003" }, + "statement": "A withheld oversight segment supports evidence of its existence, kind, and timing without revealing its content.", + "testability": "blocked", + "targets": ["oversight-profile", "trace-document", "trace-producer"], + "blocked_by": ["A10", "A17", "A18"] + } + ] +} diff --git a/conformance/requirements.schema.json b/conformance/requirements.schema.json new file mode 100644 index 0000000..d24b19f --- /dev/null +++ b/conformance/requirements.schema.json @@ -0,0 +1,94 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/requirements/0.1-draft", + "title": "PromptSyntax conformance requirement inventory", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "spec", "requirements"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "spec": { + "type": "object", + "additionalProperties": false, + "required": ["path", "commit", "declared_version", "baseline_status"], + "properties": { + "path": { "type": "string" }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "declared_version": { "type": "string" }, + "baseline_status": { "enum": ["working-draft", "frozen"] } + } + }, + "requirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/requirement" } + } + }, + "$defs": { + "requirement": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "layer", + "normative_status", + "strength", + "source", + "statement", + "testability", + "targets" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^PS-(META|CORE|CAP|EXEC|TRACE)-[A-Z0-9-]+$" + }, + "layer": { + "enum": ["meta", "core", "capabilities", "execution", "trace"] + }, + "normative_status": { "enum": ["settled", "proposed", "open"] }, + "strength": { + "enum": ["must", "must_not", "should", "should_not", "may", "invariant"] + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["path", "section", "candidate_anchor"], + "properties": { + "path": { "type": "string" }, + "section": { "type": "string" }, + "candidate_anchor": { "type": "string", "pattern": "^[a-z0-9-]+$" } + } + }, + "statement": { "type": "string", "minLength": 1 }, + "testability": { + "enum": ["automatic", "manual", "profile", "blocked"] + }, + "targets": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "corpus", + "core-parser", + "capability-document", + "execution-resolver", + "trace-document", + "trace-producer", + "interactive-ui", + "documentation", + "oversight-profile" + ] + } + }, + "blocked_by": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^A[0-9]{2}$" } + }, + "notes": { "type": "string" } + } + } + } +} diff --git a/decisions/2026-08-03-conformance-baseline-proposal.md b/decisions/2026-08-03-conformance-baseline-proposal.md new file mode 100644 index 0000000..963d543 --- /dev/null +++ b/decisions/2026-08-03-conformance-baseline-proposal.md @@ -0,0 +1,213 @@ +# Decision proposal: conformance baseline and execution-receipt outcomes + +- **Date:** 2026-08-03 +- **Status:** Proposed for v0.3 +- **Spec locations:** §4, §8, §10.5, §12, §16 OQ14 and OQ15 +- **Decision requested:** How PromptSyntax freezes a test oracle and represents exact + binding, fallback, substitution, and refusal without conflating them. + +## Context + +PromptSyntax has two behavior-compatible parser implementations and a small shared Core +fixture set, but it does not yet have a specification-owned conformance corpus. The Trace +schema describes the intended turn-level receipt while leaving several normative behaviors +to prose. A conformance runner written before those behaviors are reconciled would make the +first implementation, rather than the specification, the accidental oracle. + +The most consequential ambiguity concerns the paper's central example. The existing schema +can record `requested`, `applied`, and `bound`, but its generic fill status cannot +unambiguously distinguish: + +1. the requested model was kept; +2. a user-declared fallback step ran; +3. the venue substituted a different model outside the declared route; or +4. the request was refused before execution. + +Those are different outcomes for reliance, authorization, and conformance. A user-declared +fallback is not a silent substitution. A strict request cannot be successfully substituted. + +The audit also found mechanical mismatches. Section 10.5 requires `SAFETY_BLOCKED`, but the +failure-code registry and schema omit it. User-initiated traces normatively require compiled +prompt segments and a resolution report, but the schema makes both optional. R1 requires a +byte-exact comparison without defining the independent execution bytes against which the +trace is checked. + +## Proposed decision + +### 1. Freeze one baseline before normative vectors + +Consolidate settled v0.2 and v0.2.1 material into a clean v0.3 baseline. Proposed §13.2 +standing authoring surfaces remain outside the first v0.3 conformance profile until their +capability and trace shapes are settled. + +Every normative rule receives a stable identifier. Tests cite the identifier and immutable +specification commit. Section numbers remain explanatory and may change editorially. + +The specification is the oracle. A vector may expose an ambiguity but may not resolve one. +When independent implementations disagree and the prose does not decide the result, the +specification is amended before an expected answer is added. + +### 2. Name three different conformance targets + +1. **Corpus conformance:** the suite metadata, requirement links, artifacts, hashes, and + expectations are internally valid. +2. **Trace document conformance:** a supplied trace and its external artifacts satisfy the + structural and semantic contract. +3. **Trace producer conformance:** given an independent deterministic execution transcript, + an implementation emits a conforming trace that agrees with the transcript. + +Document conformance cannot prove what actually crossed a provider boundary. Producer +conformance supplies those facts independently to the trace producer. Neither target proves +that a self-reporting venue was truthful in deployment. + +### 3. Make entity outcomes explicit + +Replace the generic interpretation of entity fill entries with a discriminated entity +outcome whose status is exactly one of: + +- `kept`: the applied canonical entity equals the requested canonical entity; +- `fallback`: a different entity was applied through an explicit step in the authored route; +- `substituted`: a different entity was applied outside the authored route under a policy + that permits venue discretion; or +- `refused`: no entity invocation occurred. + +Every entity outcome records the requested canonical entity or route, the applied canonical +entity when one ran, the deciding fulfillment policy, and the relevant attempts. `fallback` +identifies the authored route step. `substituted` records a typed reason and deciding +authority. `refused` records a typed failure code and recourse when one exists. + +An exact entity request is strict by default. It therefore permits only `kept` or `refused`. +`substituted` under strict policy is non-conformant. An explicit authored fallback can +produce `fallback` without weakening strict binding of each individual step. + +Parameter, budget, and action fill entries retain their own status vocabularies rather than +sharing every entity outcome value. + +### 4. Extend the typed reason registry + +Add at least: + +- `ENTITY_UNAVAILABLE`: the canonical entity resolved but could not be invoked because of + capacity or availability; +- `ENTITY_SUBSTITUTED`: a different entity was applied outside the authored route; and +- `SAFETY_BLOCKED`: a safety envelope refused the request. + +`ENTITY_NOT_FOUND` remains a resolution failure and must not stand in for temporary +availability. `FALLBACK_EXHAUSTED` remains the terminal outcome after all legal route steps +fail. An explicit fallback attempt carries the reason its preceding step failed; it does not +carry `ENTITY_SUBSTITUTED` merely because the final entity differs from the first step. + +The registry covers both failures and disclosed degradations, so the specification should +rename it from "failure codes" to "outcome reason codes." + +### 5. Supply independent evidence for R1 + +Trace document validation can check segment structure, hashes, and internally materialized +content. Byte-exact producer conformance additionally receives the actual UTF-8 request +bytes from the deterministic execution transcript. The runner reconstructs bytes from the +trace and compares them with that independent value. + +The v0.3 baseline must define: + +- UTF-8 as the comparison encoding; +- no implicit Unicode normalization or line-ending conversion after compilation; +- byte lengths, not character counts; +- artifact digest and resolution behavior; and +- the tier-specific comparison value for withheld content. + +Without independent request bytes, an internally self-consistent trace can satisfy document +validation but cannot claim R1 producer conformance. + +### 6. Make schema and semantic validation boundaries explicit + +JSON Schema enforces local structure. The Rust semantic validator enforces cross-field, +graph, ordering, content, policy, and coverage invariants. Requirements that cannot be +expressed faithfully in JSON Schema remain normative and are not weakened to fit the schema. + +At minimum, v0.3 schema reconciliation must address: + +- conditional requirements for user-initiated segments and resolution reports; +- failed or blocked routing records for which no `bound` model exists; +- unique event, segment, and step identifiers; +- contiguous ordered segment indices; +- step and parent resolution within the correct inference; +- external digest syntax and non-negative byte lengths; +- materialized content requirements; +- exact extension and unknown-field policy; +- boundary calls that fail without receiving content; +- refusal authority, trigger provenance, and recourse; +- oversight retention, residency, sequence, and gap records; and +- legal combinations of policy, status, applied value, and reason code. + +### 7. Scope the first paper profile narrowly and honestly + +The minimum paper profile is: + +```text +PS/Core 0.3 + PS/Trace 0.3 +(user tier, routing complete, named assembly coverage) +``` + +It includes all settled user-tier `MUST` and `MUST NOT` requirements, R1 through R6, +requested-to-applied entity outcomes, explicit fallback, strict refusal, tool boundary +events, and deterministic producer replay. + +Developer, operator, and oversight cases may be published alongside it but cannot be +included in the headline conformance claim until each named profile is complete. Proposed +standing-authoring-surface behavior remains an extension candidate rather than silently +entering Core conformance. + +## Alternatives considered + +### Let the Rust reference implementation define ambiguous behavior + +Rejected. This creates implementation compatibility, not specification conformance, and +makes independent implementation evidence circular. + +### Treat every requested/applied mismatch as substitution + +Rejected. It falsely classifies a user-authored fallback as venue discretion and erases the +authorization information carried by the route. + +### Infer substitution from string inequality without an explicit outcome + +Rejected. It cannot distinguish aliases, declared fallbacks, normalization, or a dishonest +`filled` label, and it gives user interfaces no stable semantic field to present. + +### Use only JSON Schema + +Rejected. JSON Schema does not establish graph acyclicity, byte-exact reconstruction, +cross-event completeness, or fulfillment-policy consistency. + +### Exercise live provider APIs in normative tests + +Rejected. Provider behavior, availability, pricing, and model aliases are unstable. Live +experiments may test venue profiles but cannot define deterministic core conformance. + +### Copy Prompty's runners + +Rejected. PromptSyntax borrows the specification-owned vector architecture but implements a +typed Rust runner against PromptSyntax requirements. No Prompty runtime is required. + +## Consequences + +- The existing v0.2 Trace schema must change before a normative v0.3 suite can be released. +- Some currently schema-valid documents will become invalid because outcome states and + required fields become explicit. +- A trace can pass document validation while lacking producer evidence for R1. Reports must + name the target they passed. +- The paper can evaluate kept versus substituted outcomes without mislabeling explicit + fallback. +- Two author-controlled implementations demonstrate specification precision, not + independent certification. + +## Questions for PR review + +1. Should `substituted` ever be legal under an explicitly selected `best-effort` entity + policy, or should entity substitution always require pre-execution confirmation? +2. Should the user-tier receipt expose the deciding authority for capacity substitution as + `venue`, `operator`, or another explicit authority value? +3. At a tier where content is withheld, should R1 compare the original compiled bytes using + oversight-only evidence, or compare a canonical typed placeholder visible at that tier? +4. Should unknown JSON fields be rejected by default with namespaced extension objects, or + preserved for forward compatibility? diff --git a/decisions/README.md b/decisions/README.md index 8cf6f8c..6e157f5 100644 --- a/decisions/README.md +++ b/decisions/README.md @@ -14,5 +14,6 @@ | 10 | Oversight | Evidence substrate + external profiles | Fixed audit tier; jurisdiction rules in core | SPEC §12.3 | | 11 | Policy systems | PEP/PDP seam + external bindings | Policy language in core | SPEC §10.4 | | 12 | Standing promotion (proposed) | Versioned, origin-bound declared authoring surfaces | Parse-all output; delimiter-only promotion; envelope widening | SPEC §13.2 | +| 13 | Conformance baseline (proposed) | Spec-owned vectors, Rust tooling, explicit entity outcomes | Implementation-defined oracle; generic requested/applied mismatch | SPEC §4, §8, §12 | Full rationale: the spec's decision-record notes and the project's design documents. diff --git a/examples/prompt-trace-0.3-draft.example.json b/examples/prompt-trace-0.3-draft.example.json new file mode 100644 index 0000000..6abc7a4 --- /dev/null +++ b/examples/prompt-trace-0.3-draft.example.json @@ -0,0 +1,87 @@ +{ + "ps_trace": "0.3-draft", + "turn": "turn-example-001", + "tier": "user", + "mode": "executed", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "vendor_profile": "vendor-profile/example-2026-08" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "events": [ + { + "id": "inference-001", + "event": "inference", + "kind": "user-initiated", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "outcome": "invoked", + "bound": "model:example/atlas-4@2026-08-01", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { + "cost_usd": 0.041, + "wall_time_ms": 3210 + } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "applied": "model:example/atlas-4@2026-08-01", + "status": "kept", + "policy": "strict" + }, + { + "kind": "parameter", + "name": "temperature", + "requested": 0.2, + "applied": 0.2, + "status": "applied", + "policy": "strict" + } + ] + }, + "segments": [ + { + "i": 0, + "id": "segment-user-001", + "origin": "user-authored", + "content": { + "state": "inline", + "text": "Summarize the report." + }, + "step": "step-user-001" + } + ], + "steps": [ + { + "id": "step-user-001", + "kind": "user-content", + "tier": "user" + } + ], + "resolution": { + "bindings": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "refusals": [] + } + } + ] +} diff --git a/schemas/prompt-trace-0.3-draft.schema.json b/schemas/prompt-trace-0.3-draft.schema.json new file mode 100644 index 0000000..6690afc --- /dev/null +++ b/schemas/prompt-trace-0.3-draft.schema.json @@ -0,0 +1,643 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/schemas/prompt-trace/0.3-draft", + "title": "Prompt Trace 0.3 candidate", + "description": "Non-normative candidate schema used to resolve the v0.2 conformance audit. It must not be used for an unqualified conformance claim.", + "type": "object", + "additionalProperties": false, + "required": ["ps_trace", "turn", "tier", "mode", "venue", "coverage", "events"], + "properties": { + "ps_trace": { "const": "0.3-draft" }, + "turn": { "type": "string", "minLength": 1 }, + "tier": { "enum": ["user", "developer", "operator", "oversight"] }, + "mode": { "enum": ["dry-run", "executed"] }, + "venue": { "$ref": "#/$defs/venue" }, + "coverage": { "$ref": "#/$defs/coverage" }, + "integrity": { "$ref": "#/$defs/integrity" }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { "$ref": "#/$defs/inference" }, + { "$ref": "#/$defs/boundary_event" } + ] + } + }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "$defs": { + "venue": { + "type": "object", + "additionalProperties": false, + "required": ["id", "capability_document"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "capability_document": { "type": "string", "format": "uri" }, + "vendor_profile": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["routing", "assembly", "inline_threshold_bytes"], + "properties": { + "routing": { "const": "complete" }, + "assembly": { "enum": ["complete", "user-initiated-only"] }, + "inline_threshold_bytes": { "type": "integer", "minimum": 4096 }, + "profile": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "inference": { + "type": "object", + "additionalProperties": false, + "required": ["id", "event", "kind", "routing", "segments", "steps", "resolution"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "event": { "const": "inference" }, + "kind": { "enum": ["user-initiated", "internal-continuation"] }, + "routing": { "$ref": "#/$defs/routing_record" }, + "segments": { + "oneOf": [ + { "$ref": "#/$defs/segment_list" }, + { "$ref": "#/$defs/segments_omitted" } + ] + }, + "steps": { + "type": "array", + "items": { "$ref": "#/$defs/step" } + }, + "resolution": { "$ref": "#/$defs/resolution_report" }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["kind"], + "properties": { "kind": { "const": "user-initiated" } } + }, + "then": { + "properties": { "segments": { "$ref": "#/$defs/segment_list" } } + } + } + ] + }, + + "routing_record": { + "type": "object", + "additionalProperties": false, + "required": ["requested", "policy", "outcome", "attempts", "fill"], + "properties": { + "requested": { + "type": "string", + "minLength": 1, + "description": "Authored directive text or synthesized canonical route." + }, + "policy": { "enum": ["strict", "partial", "best-effort"] }, + "outcome": { "enum": ["invoked", "refused"] }, + "bound": { + "type": "string", + "minLength": 1, + "description": "Canonical pinned entity actually invoked." + }, + "attempts": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/attempt" } + }, + "fill": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/fill_entry" } + }, + "budget": { "$ref": "#/$defs/budget_echo" }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["outcome"], + "properties": { "outcome": { "const": "invoked" } } + }, + "then": { "required": ["bound"] }, + "else": { "not": { "required": ["bound"] } } + } + ] + }, + + "attempt": { + "type": "object", + "additionalProperties": false, + "required": ["ref", "outcome"], + "properties": { + "ref": { "type": "string", "minLength": 1 }, + "bound": { "type": "string", "minLength": 1 }, + "outcome": { "enum": ["filled", "failed", "blocked"] }, + "reasons": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/outcome_reason" } + }, + "measured": { "$ref": "#/$defs/measured" }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["outcome"], + "properties": { "outcome": { "const": "filled" } } + }, + "then": { "required": ["bound", "measured"] }, + "else": { "required": ["reasons"] } + } + ] + }, + + "fill_entry": { + "oneOf": [ + { "$ref": "#/$defs/entity_fill" }, + { "$ref": "#/$defs/parameter_fill" }, + { "$ref": "#/$defs/budget_fill" }, + { "$ref": "#/$defs/action_fill" } + ] + }, + + "entity_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "requested", "status", "policy"], + "properties": { + "kind": { "const": "entity" }, + "requested": { "type": "string", "minLength": 1 }, + "applied": { "type": "string", "minLength": 1 }, + "status": { "enum": ["kept", "fallback", "substituted", "refused"] }, + "policy": { "enum": ["strict", "partial", "best-effort"] }, + "route_step": { "type": "integer", "minimum": 0 }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "deciding_authority": { "$ref": "#/$defs/authority" }, + "mapping_rule": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "enum": ["kept", "fallback", "substituted"] } } + }, + "then": { "required": ["applied"] }, + "else": { + "required": ["reason", "deciding_authority"], + "not": { "required": ["applied"] } + } + }, + { + "if": { + "required": ["status"], + "properties": { "status": { "const": "fallback" } } + }, + "then": { "required": ["route_step"] } + }, + { + "if": { + "required": ["status"], + "properties": { "status": { "const": "substituted" } } + }, + "then": { + "required": ["reason", "deciding_authority"], + "properties": { "reason": { "const": "ENTITY_SUBSTITUTED" } } + } + } + ] + }, + + "parameter_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "name", "requested", "status", "policy"], + "properties": { + "kind": { "const": "parameter" }, + "name": { "type": "string", "minLength": 1 }, + "requested": true, + "normalized": true, + "applied": true, + "status": { + "enum": ["applied", "translated", "clamped", "unsupported", "ignored", "refused"] + }, + "policy": { "enum": ["strict", "partial", "best-effort"] }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "mapping_rule": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "enum": ["applied", "translated", "clamped"] } } + }, + "then": { "required": ["applied"] }, + "else": { "required": ["reason"] } + }, + { + "if": { + "required": ["status"], + "properties": { "status": { "enum": ["translated", "clamped"] } } + }, + "then": { "required": ["normalized", "mapping_rule"] } + } + ] + }, + + "budget_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "name", "requested", "status"], + "properties": { + "kind": { "const": "budget" }, + "name": { "type": "string", "minLength": 1 }, + "requested": true, + "applied": true, + "status": { "enum": ["applied", "exceeded", "unsupported", "refused"] }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "const": "applied" } } + }, + "then": { "required": ["applied"] }, + "else": { "required": ["reason"] } + } + ] + }, + + "action_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "requested", "status"], + "properties": { + "kind": { "const": "action" }, + "requested": { "type": "string", "minLength": 1 }, + "applied": { "type": "string", "minLength": 1 }, + "status": { "enum": ["applied", "refused"] }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "deciding_authority": { "$ref": "#/$defs/authority" }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "const": "applied" } } + }, + "then": { "required": ["applied"] }, + "else": { "required": ["reason", "deciding_authority"] } + } + ] + }, + + "outcome_reason": { + "enum": [ + "ENTITY_NOT_FOUND", + "ENTITY_AMBIGUOUS", + "ENTITY_UNAVAILABLE", + "ENTITY_SUBSTITUTED", + "ENTITY_UNAUTHORIZED", + "PARAM_UNSUPPORTED", + "PARAM_CLAMPED", + "PARAM_IGNORED", + "BUDGET_PREFLIGHT_REJECTED", + "BUDGET_EXCEEDED", + "TIMEOUT_NOT_CANCELLED", + "DATA_POLICY_BLOCKED", + "SAFETY_BLOCKED", + "SPAN_EXECUTION_UNSUPPORTED", + "VERSION_UNSUPPORTED", + "FALLBACK_EXHAUSTED" + ] + }, + + "authority": { + "enum": [ + "law-regulation", + "platform", + "venue-operations", + "organization", + "application", + "user", + "prompt" + ] + }, + + "budget_echo": { + "type": "object", + "additionalProperties": false, + "properties": { + "estimated_cost_max_usd": { "type": "number", "minimum": 0 }, + "observed_cost_max_usd": { "type": "number", "minimum": 0 }, + "wall_time_max_ms": { "type": "integer", "minimum": 0 }, + "first_token_max_ms": { "type": "integer", "minimum": 0 }, + "cancellation": { "enum": ["required", "preferred", "unsupported"] }, + "scope": { "enum": ["attempt", "chain"] }, + "set_by_authority": { "$ref": "#/$defs/authority" }, + "measurement_profile": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "measured": { + "type": "object", + "additionalProperties": false, + "properties": { + "cost_usd": { "type": "number", "minimum": 0 }, + "wall_time_ms": { "type": "integer", "minimum": 0 }, + "first_token_ms": { "type": "integer", "minimum": 0 }, + "cancelled": { "type": "boolean" }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "segment_list": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/segment" } + }, + + "segment": { + "type": "object", + "additionalProperties": false, + "required": ["i", "origin", "content"], + "properties": { + "i": { "type": "integer", "minimum": 0 }, + "id": { "type": "string", "minLength": 1 }, + "origin": { "enum": ["user-authored", "injected"] }, + "content": { "$ref": "#/$defs/content" }, + "step": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["origin"], + "properties": { "origin": { "const": "injected" } } + }, + "then": { "required": ["step"] } + } + ] + }, + + "content": { + "oneOf": [ + { "$ref": "#/$defs/content_inline" }, + { "$ref": "#/$defs/content_external" }, + { "$ref": "#/$defs/content_withheld" } + ] + }, + + "content_inline": { + "type": "object", + "additionalProperties": false, + "required": ["state", "text"], + "properties": { + "state": { "const": "inline" }, + "text": { "type": "string" }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "content_external": { + "type": "object", + "additionalProperties": false, + "required": ["state", "sha256", "length", "handle"], + "properties": { + "state": { "const": "external" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "length": { "type": "integer", "minimum": 0 }, + "media": { "type": "string", "minLength": 1 }, + "handle": { "type": "string", "format": "uri" }, + "materialized": { "$ref": "#/$defs/materialized" }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "materialized": { + "type": "object", + "additionalProperties": false, + "required": ["text", "materialized_at"], + "properties": { + "text": { "type": "string" }, + "materialized_at": { "type": "string", "format": "date-time" }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "content_withheld": { + "type": "object", + "additionalProperties": false, + "required": ["state", "digest", "length_band", "tier", "kind"], + "properties": { + "state": { "const": "withheld" }, + "digest": { "type": "string", "minLength": 1 }, + "length_band": { "enum": ["<1KB", "1-4KB", "4-16KB", "16-64KB", ">64KB"] }, + "tier": { "enum": ["developer", "operator", "oversight"] }, + "kind": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "segments_omitted": { + "type": "object", + "additionalProperties": false, + "required": ["coverage", "reason", "declared_in"], + "properties": { + "coverage": { "const": "not-provided" }, + "reason": { "enum": ["should-level-not-implemented", "tier-excluded"] }, + "declared_in": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "step": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { + "enum": [ + "user-content", + "context-inclusion", + "retrieval", + "skill-invocation", + "template-expansion", + "policy-injection", + "system-injection", + "tool-result-inclusion", + "loop-instruction", + "truncation", + "transformation", + "promotion" + ] + }, + "via": { "type": "string", "minLength": 1 }, + "of": { "type": "string", "minLength": 1 }, + "parent": { "type": "string", "minLength": 1 }, + "tier": { "enum": ["user", "developer", "operator", "oversight"] }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "resolution_report": { + "type": "object", + "additionalProperties": false, + "required": ["bindings", "refusals"], + "properties": { + "bindings": { + "type": "array", + "items": { "$ref": "#/$defs/binding" } + }, + "refusals": { + "type": "array", + "items": { "$ref": "#/$defs/refusal" } + }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "binding": { + "type": "object", + "additionalProperties": false, + "required": ["ref", "bound", "rule", "ambiguity_surfaced"], + "properties": { + "ref": { "type": "string", "minLength": 1 }, + "bound": { "type": "string", "minLength": 1 }, + "rule": { "type": "string", "minLength": 1 }, + "ambiguity_surfaced": { "type": "boolean" }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "refusal": { + "type": "object", + "additionalProperties": false, + "required": ["ref", "reason", "authority", "recourse"], + "properties": { + "ref": { "type": "string", "minLength": 1 }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "authority": { "$ref": "#/$defs/authority" }, + "triggering_step": { "type": "string", "minLength": 1 }, + "recourse": { "$ref": "#/$defs/recourse" }, + "policy_decision_id": { "type": "string", "minLength": 1 }, + "policy_bundle": { "type": "string", "minLength": 1 }, + "policy_engine": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["reason"], + "properties": { "reason": { "const": "SAFETY_BLOCKED" } } + }, + "then": { "required": ["triggering_step"] } + } + ] + }, + + "recourse": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["available", "kind"], + "properties": { + "available": { "const": true }, + "kind": { "enum": ["appeal", "verification", "feedback", "retry"] }, + "uri": { "type": "string", "format": "uri" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["available", "reason"], + "properties": { + "available": { "const": false }, + "reason": { "type": "string", "minLength": 1 } + } + } + ] + }, + + "boundary_event": { + "type": "object", + "additionalProperties": false, + "required": ["id", "event", "tool", "outcome", "sent", "fill", "measured"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "event": { "const": "boundary" }, + "tool": { "type": "string", "minLength": 1 }, + "outcome": { "enum": ["succeeded", "failed", "blocked", "cancelled"] }, + "sent": { "$ref": "#/$defs/content" }, + "received": { "$ref": "#/$defs/content" }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "fill": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/fill_entry" } + }, + "internal": { "const": "outside-trace-perimeter" }, + "measured": { "$ref": "#/$defs/measured" }, + "extensions": { "$ref": "#/$defs/extensions" } + }, + "allOf": [ + { + "if": { + "required": ["outcome"], + "properties": { "outcome": { "const": "succeeded" } } + }, + "then": { "required": ["received"] }, + "else": { "required": ["reason"] } + } + ] + }, + + "integrity": { + "type": "object", + "additionalProperties": false, + "properties": { + "algorithm": { "const": "sha256-chain" }, + "event_hashes": { + "type": "array", + "items": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": ["alg", "key_id", "value"], + "properties": { + "alg": { "type": "string", "minLength": 1 }, + "key_id": { "type": "string", "minLength": 1 }, + "value": { "type": "string", "minLength": 1 }, + "timestamp": { "type": "string", "format": "date-time" }, + "timestamp_authority": { "type": "string", "minLength": 1 } + } + }, + "completeness_attestation": { "type": "string", "minLength": 1 }, + "extensions": { "$ref": "#/$defs/extensions" } + } + }, + + "extensions": { + "type": "object", + "propertyNames": { + "pattern": "^[a-z0-9][a-z0-9.-]*:[a-zA-Z0-9_.-]+$" + }, + "additionalProperties": true + } + } +} From 797421238e5cfeef8bab182cd5eeb486a145dde2 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 01:27:49 +0700 Subject: [PATCH 2/5] feat: add executable Trace conformance suite --- .github/workflows/conformance.yml | 101 ++ .gitignore | 1 + README.md | 22 +- biome.json | 11 +- conformance/README.md | 50 +- conformance/adapter-result.schema.json | 39 + conformance/adapters/README.md | 14 + conformance/case-family.schema.json | 84 ++ conformance/cases/core-parser.json | 114 ++ conformance/core-cases.schema.json | 39 + conformance/diagnostic-registry.json | 318 +++++ .../families/trace-boundary-tamper.json | 160 +++ conformance/families/trace-outcomes.json | 319 +++++ .../families/trace-producer-replay.json | 214 ++++ conformance/families/trace-structure.json | 248 ++++ conformance/fixtures/trace-user-kept.json | 88 ++ .../fixtures/transcript-user-kept.json | 16 + conformance/implementations.lock.json | 21 + conformance/implementations.schema.json | 32 + conformance/manifest.json | 68 +- conformance/manifest.schema.json | 35 +- conformance/profile.schema.json | 77 ++ conformance/profiles/README.md | 40 + .../profiles/trace-user-0.3-draft.json | 53 + conformance/releases/0.1.0-rc.1.md | 30 + conformance/report.schema.json | 131 +++ conformance/reports/0.1.0-rc.1.json | 602 ++++++++++ conformance/requirements.json | 378 +++++- conformance/runner/Cargo.lock | 1041 +++++++++++++++++ conformance/runner/Cargo.toml | 25 + conformance/runner/README.md | 31 + conformance/runner/src/lib.rs | 694 +++++++++++ conformance/runner/src/main.rs | 120 ++ conformance/runner/src/suite.rs | 967 +++++++++++++++ conformance/runner/src/trace.rs | 942 +++++++++++++++ conformance/transcript.schema.json | 73 ++ ...026-08-03-conformance-baseline-proposal.md | 8 +- examples/prompt-trace-0.3-draft.example.json | 1 + schemas/prompt-trace-0.3-draft.schema.json | 13 +- 39 files changed, 7121 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/conformance.yml create mode 100644 conformance/adapter-result.schema.json create mode 100644 conformance/adapters/README.md create mode 100644 conformance/case-family.schema.json create mode 100644 conformance/cases/core-parser.json create mode 100644 conformance/core-cases.schema.json create mode 100644 conformance/families/trace-boundary-tamper.json create mode 100644 conformance/families/trace-outcomes.json create mode 100644 conformance/families/trace-producer-replay.json create mode 100644 conformance/families/trace-structure.json create mode 100644 conformance/fixtures/trace-user-kept.json create mode 100644 conformance/fixtures/transcript-user-kept.json create mode 100644 conformance/implementations.lock.json create mode 100644 conformance/implementations.schema.json create mode 100644 conformance/profile.schema.json create mode 100644 conformance/profiles/README.md create mode 100644 conformance/profiles/trace-user-0.3-draft.json create mode 100644 conformance/releases/0.1.0-rc.1.md create mode 100644 conformance/report.schema.json create mode 100644 conformance/reports/0.1.0-rc.1.json create mode 100644 conformance/runner/Cargo.lock create mode 100644 conformance/runner/Cargo.toml create mode 100644 conformance/runner/README.md create mode 100644 conformance/runner/src/lib.rs create mode 100644 conformance/runner/src/main.rs create mode 100644 conformance/runner/src/suite.rs create mode 100644 conformance/runner/src/trace.rs create mode 100644 conformance/transcript.schema.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..4af8308 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,101 @@ +name: Conformance + +on: + pull_request: + paths: + - ".github/workflows/conformance.yml" + - "conformance/**" + - "decisions/**" + - "examples/**" + - "schemas/**" + - "spec/**" + push: + branches: + - master + paths: + - ".github/workflows/conformance.yml" + - "conformance/**" + - "decisions/**" + - "examples/**" + - "schemas/**" + - "spec/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + suite: + name: Trace suite + runs-on: ubuntu-latest + steps: + - name: Checkout standard and corpus + uses: actions/checkout@v4 + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.85.0 + components: rustfmt, clippy + + - name: Check Rust formatting + run: cargo fmt --manifest-path conformance/runner/Cargo.toml -- --check + + - name: Run strict Rust lint + run: cargo clippy --manifest-path conformance/runner/Cargo.toml --all-targets -- -D warnings + + - name: Run runner tests and suite + run: cargo test --locked --manifest-path conformance/runner/Cargo.toml + + - name: Verify deterministic report snapshot + run: | + cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-suite . > /tmp/promptsyntax-conformance-report.json + cmp conformance/reports/0.1.0-rc.1.json /tmp/promptsyntax-conformance-report.json + + cross-language-core: + name: Rust and TypeScript Core adapters + runs-on: ubuntu-latest + steps: + - name: Checkout standard and canonical cases + uses: actions/checkout@v4 + with: + path: standard + + - name: Checkout pinned Rust implementation + uses: actions/checkout@v4 + with: + repository: pathscale/PromptSyntax-rs + ref: c64072f618ffdb53cc389fc40dfe0370c4cf05be + path: implementations/promptsyntax-rs + + - name: Checkout pinned TypeScript implementation + uses: actions/checkout@v4 + with: + repository: pathscale/PromptSyntax-ts + ref: b22acfac066c0ea6eec7d042d69040c695ef5422 + path: implementations/promptsyntax-ts + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.85.0 + + - name: Install Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + + - name: Run Rust adapter against canonical cases + run: | + cp standard/conformance/cases/core-parser.json implementations/promptsyntax-rs/tests/conformance.json + cargo test --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml + + - name: Install TypeScript implementation dependencies + run: bun install --frozen-lockfile + working-directory: implementations/promptsyntax-ts + + - name: Run TypeScript adapter against canonical cases + run: | + cp ../../standard/conformance/cases/core-parser.json test/conformance.json + bun test + working-directory: implementations/promptsyntax-ts diff --git a/.gitignore b/.gitignore index 7ad24ee..645cec4 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json .eslintcache .cache *.tsbuildinfo +conformance/runner/target # LaTeX build artifacts (paper/) *.aux diff --git a/README.md b/README.md index 7860b11..ebf50a1 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,14 @@ anything. | [`examples/`](./examples/) | Machine-validated example trace | | [`decisions/`](./decisions/) | Design decision records (with rejected alternatives) | | [`profiles/`](./profiles/) | Externalized mappings: vendor, oversight, policy bindings | +| [`conformance/`](./conformance/) | Language-independent requirements, vectors, schemas, reports, and Rust runner | ## Status -**Draft v0.2.1, pre-implementation. Not yet stable.** A reference implementation and a -formative multi-stakeholder study are in progress; an academic paper is in preparation. -This is a proposal seeking scrutiny, not a finished standard. +**Draft v0.2.1 with experimental implementations. Not yet stable.** Rust and TypeScript +Core parsers and a candidate Trace conformance suite are under active review; a formative +multi-stakeholder study and academic paper are in preparation. This is a proposal seeking +scrutiny, not a finished standard. ## Design principles @@ -72,6 +74,20 @@ Per-layer, versioned, and named, e.g. `PS/Core 0.2 + PS/Trace 0.2 (user tier, routing-complete/assembly-partial)`. **Unqualified conformance claims are non-conformant:** an implementation must state what it covers. +The draft suite is self-contained in this repository. Its stable-Rust runner has no +provider, model, Node, or Python dependency: + +```sh +cargo test --manifest-path conformance/runner/Cargo.toml +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + check-requirements conformance/requirements.json +cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . +``` + +Candidate `0.1.0-rc.1` currently executes 34 Trace document and producer cases with full +coverage of its explicitly named 22-requirement profile. Candidate status is not a released +or unqualified implementation-conformance claim. + ## Contributing This is a draft seeking exactly the scrutiny that hardens a specification. Feedback is diff --git a/biome.json b/biome.json index afc341c..80580b3 100644 --- a/biome.json +++ b/biome.json @@ -60,6 +60,15 @@ } }, "files": { - "includes": ["**", "!dist", "!node_modules", "!*.config.js", "!*.config.ts", "!.claude"] + "includes": [ + "**", + "!dist", + "!node_modules", + "!*.config.js", + "!*.config.ts", + "!.claude", + "!conformance/reports", + "!conformance/runner/target" + ] } } diff --git a/conformance/README.md b/conformance/README.md index 9c97d53..dda3d58 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -1,30 +1,54 @@ # PromptSyntax conformance corpus -This directory will contain the canonical, language-independent PromptSyntax conformance -corpus. The specification and expected vectors are the oracle. Implementations consume the -corpus but do not define its expected outcomes. +This directory contains the canonical, language-independent PromptSyntax conformance +corpus and its repository-owned stable-Rust runner. The specification and expected vectors +are the oracle. Implementations consume the corpus but do not define its expected outcomes. ## Current status -The corpus is being bootstrapped against the specification draft at commit -`7456634817f4eb68c8909a58b476505e6fd7c063`. It is not yet a released conformance suite and -must not be used for an unqualified conformance claim. +Version `0.1.0-rc.1` is a candidate suite against the specification draft at commit +`7456634817f4eb68c8909a58b476505e6fd7c063`. It executes 34 document and producer cases +across four Trace families and covers all 22 requirements in its named candidate profile. +It is not a released conformance suite and must not be used for an unqualified claim. -Normative vectors are blocked until the v0.3 baseline reconciles the specification and Trace -schema. The initial machine-readable requirement inventory may contain blocked or proposed -entries so that gaps remain visible rather than being silently omitted. +The cases freeze provisional v0.3 decisions for review without pretending those decisions +are already normative. The machine-readable inventory retains blocked, proposed, and open +entries so unresolved specification gaps remain visible rather than being silently omitted. -## Planned ownership +## Ownership -- `promptsyntax.org` owns specifications, schemas, requirements, vectors, and published - reports. -- `PromptSyntax-rs` owns the stable-Rust validator and conformance runner. +- `promptsyntax.org` owns specifications, schemas, requirements, vectors, the stable-Rust + runner, and published reports so the complete suite is versioned atomically. +- `PromptSyntax-rs` and `PromptSyntax-ts` are implementations under test, not sources of + expected answers. - Each implementation exposes a thin adapter that contains no expected answers. The normative runner does not require Python, Node, Bun, a provider SDK, a network connection, or model credentials. The TypeScript implementation may be invoked separately with Bun when producing cross-language evidence. +Run the repository-owned checker from the repository root: + +```bash +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + check-requirements conformance/requirements.json + +cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . +``` + +The command emits the deterministic report stored in +`conformance/reports/0.1.0-rc.1.json`. CI regenerates it and requires a byte-for-byte match. + +## Layout + +- `profiles/` freezes the named scope and candidate semantic decisions. +- `families/` contains language-independent cases and expected diagnostics. +- `fixtures/` contains reusable Trace and deterministic transcript inputs. +- `cases/core-parser.json` is the single Core fixture consumed by both parser adapters. +- `reports/` contains deterministic, machine-readable evidence snapshots. +- `runner/` contains the independent stable-Rust checker. +- `adapters/` defines the implementation boundary and forbids embedded expected answers. + ## Conformance targets 1. **Corpus:** metadata, references, artifacts, hashes, and coverage are internally valid. diff --git a/conformance/adapter-result.schema.json b/conformance/adapter-result.schema.json new file mode 100644 index 0000000..b5e2059 --- /dev/null +++ b/conformance/adapter-result.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/adapter-result/0.1-draft", + "title": "PromptSyntax implementation adapter result", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "target", "implementation", "results"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "target": { "enum": ["core-parser", "trace-producer"] }, + "implementation": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "commit"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" } + } + }, + "results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["case_id", "conformant", "diagnostics", "output"], + "properties": { + "case_id": { "type": "string", "minLength": 1 }, + "conformant": { "type": "boolean" }, + "diagnostics": { + "type": "array", + "items": { "type": "string" } + }, + "output": true + } + } + } + } +} diff --git a/conformance/adapters/README.md b/conformance/adapters/README.md new file mode 100644 index 0000000..0e7e3da --- /dev/null +++ b/conformance/adapters/README.md @@ -0,0 +1,14 @@ +# Implementation adapters + +The suite owns expected answers. An implementation adapter only converts a canonical case +into an implementation call and converts the result into the language-neutral adapter +result shape. Adapters must not contain alternate expectations or silently skip cases. + +The current Rust and TypeScript parser tests already implement the Core adapter contract: +they consume the same fields in `conformance/cases/core-parser.json` and compare round-trip +text, data-plane text, directive kinds, and diagnostic codes. CI injects that one canonical +file into both implementation checkouts before running their native test commands. + +A Trace producer adapter receives deterministic transcript facts and emits a Prompt Trace. +The repository-owned runner then evaluates the emitted trace. Provider APIs and live model +calls are outside the normative adapter protocol because they are not deterministic. diff --git a/conformance/case-family.schema.json b/conformance/case-family.schema.json new file mode 100644 index 0000000..2b836fa --- /dev/null +++ b/conformance/case-family.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/case-family/0.1-draft", + "title": "PromptSyntax conformance case family", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "id", "version", "status", "profile", "target", "cases"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, + "version": { "type": "string", "minLength": 1 }, + "status": { "enum": ["candidate", "normative"] }, + "profile": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, + "target": { "enum": ["trace-document", "trace-producer"] }, + "cases": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/case" } + } + }, + "$defs": { + "case": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "requirements", "trace", "expected"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, + "description": { "type": "string", "minLength": 1 }, + "requirements": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^PS-[A-Z0-9-]+$" } + }, + "trace": { "type": "string", "minLength": 1 }, + "patch": { + "type": "array", + "items": { "$ref": "#/$defs/patch_operation" } + }, + "transcript": { "type": "string", "minLength": 1 }, + "transcript_patch": { + "type": "array", + "items": { "$ref": "#/$defs/patch_operation" } + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": ["conformant", "diagnostics"], + "properties": { + "conformant": { "type": "boolean" }, + "diagnostics": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" } + } + } + } + } + }, + "patch_operation": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["op", "path", "value"], + "properties": { + "op": { "enum": ["add", "replace"] }, + "path": { "type": "string", "pattern": "^/" }, + "value": true + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["op", "path"], + "properties": { + "op": { "const": "remove" }, + "path": { "type": "string", "pattern": "^/" } + } + } + ] + } + } +} diff --git a/conformance/cases/core-parser.json b/conformance/cases/core-parser.json new file mode 100644 index 0000000..4b99865 --- /dev/null +++ b/conformance/cases/core-parser.json @@ -0,0 +1,114 @@ +[ + { + "name": "plain text", + "source": "Summarize this.", + "options": {}, + "data_plane": "Summarize this.", + "directives": [], + "diagnostics": [] + }, + { + "name": "qualified reference", + "source": "Read @file:q3-report.md", + "options": {}, + "data_plane": "Read ", + "directives": ["reference"], + "diagnostics": [] + }, + { + "name": "declared bare entity and action", + "source": "@opus Summarize /concise", + "options": { "entities": ["opus"], "actions": ["concise"] }, + "data_plane": " Summarize ", + "directives": ["reference", "action"], + "diagnostics": [] + }, + { + "name": "unknown bare tokens stay inert", + "source": "@unknown and /usr/bin", + "options": {}, + "data_plane": "@unknown and /usr/bin", + "directives": [], + "diagnostics": [] + }, + { + "name": "email and escapes stay inert", + "source": "mail a@b.com and \\@model:x", + "options": {}, + "data_plane": "mail a@b.com and \\@model:x", + "directives": [], + "diagnostics": [] + }, + { + "name": "fenced syntax stays inert", + "source": "```ps\n@model:openai/gpt-5.6\n```", + "options": {}, + "data_plane": "```ps\n@model:openai/gpt-5.6\n```", + "directives": [], + "diagnostics": [] + }, + { + "name": "full width sigils and Unicode identifiers", + "source": "＠file:របាយការណ៍.md ／បកប្រែ", + "options": { "actions": ["បកប្រែ"] }, + "data_plane": " ", + "directives": ["reference", "action"], + "diagnostics": [] + }, + { + "name": "budgeted fallback route", + "source": "@opus! limit(wall_time: 5s) else @model:openai/gpt-5.6@2026-07-01 else ask\nDo it", + "options": { "entities": ["opus"] }, + "data_plane": "\nDo it", + "directives": ["route"], + "diagnostics": [] + }, + { + "name": "normal span keeps authored inner data", + "source": "Before use @file:terms.md here after", + "options": {}, + "data_plane": "Before use here after", + "directives": ["span", "reference"], + "diagnostics": [] + }, + { + "name": "declared AgencyZero authoring segment", + "source": "Before\n\nAfter", + "options": { "authoring_namespaces": ["agency"] }, + "data_plane": "Before\n\nAfter", + "directives": ["authoring_segment"], + "diagnostics": [] + }, + { + "name": "undeclared authoring segment stays inert", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": ["UNCLOSED_SPAN"] + }, + { + "name": "strict frontmatter", + "source": "---ps\nversion: \"0.2\"\n---\nSummarize.", + "options": {}, + "data_plane": "Summarize.", + "directives": ["frontmatter"], + "diagnostics": [] + }, + { + "name": "bidi control rejects partial parse", + "source": "@model:openai/gpt\u202e-5", + "options": {}, + "data_plane": "@model:openai/gpt\u202e-5", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "name": "malformed qualified reference", + "source": "Use @model: now", + "options": {}, + "data_plane": "Use @model: now", + "directives": [], + "diagnostics": ["SYNTAX_INVALID"] + } +] diff --git a/conformance/core-cases.schema.json b/conformance/core-cases.schema.json new file mode 100644 index 0000000..cf170d2 --- /dev/null +++ b/conformance/core-cases.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/core-cases/0.1-draft", + "title": "PromptSyntax Core parser cases", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "source", "options", "data_plane", "directives", "diagnostics"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "source": { "type": "string" }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "entities": { "$ref": "#/$defs/string_set" }, + "actions": { "$ref": "#/$defs/string_set" }, + "authoring_namespaces": { "$ref": "#/$defs/string_set" } + } + }, + "data_plane": { "type": "string" }, + "directives": { "$ref": "#/$defs/string_list" }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + }, + "$defs": { + "string_list": { + "type": "array", + "items": { "type": "string" } + }, + "string_set": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string" } + } + } +} diff --git a/conformance/diagnostic-registry.json b/conformance/diagnostic-registry.json index ff33ab6..c7013e8 100644 --- a/conformance/diagnostic-registry.json +++ b/conformance/diagnostic-registry.json @@ -144,6 +144,324 @@ "phase": "io", "severity": "error", "description": "The schema file could not be read." + }, + { + "code": "CASE_PATCH_INVALID", + "phase": "corpus", + "severity": "error", + "description": "A case JSON Patch operation is invalid or cannot be applied to its fixture." + }, + { + "code": "SUITE_BASELINE_STATUS_INVALID", + "phase": "corpus", + "severity": "error", + "description": "Suite and specification baseline statuses form an invalid release combination." + }, + { + "code": "SUITE_CASE_ID_DUPLICATE", + "phase": "corpus", + "severity": "error", + "description": "A case identifier occurs more than once across the suite." + }, + { + "code": "SUITE_CASE_REQUIREMENT_OUTSIDE_PROFILE", + "phase": "corpus", + "severity": "error", + "description": "A case cites a requirement outside the active conformance profile." + }, + { + "code": "SUITE_CASE_REQUIREMENT_UNKNOWN", + "phase": "corpus", + "severity": "error", + "description": "A case cites a requirement absent from the requirements inventory." + }, + { + "code": "SUITE_CLAIM_FAMILY_UNKNOWN", + "phase": "corpus", + "severity": "error", + "description": "A conformance claim cites a family absent from the suite manifest." + }, + { + "code": "SUITE_CLAIM_PROFILE_MISMATCH", + "phase": "corpus", + "severity": "error", + "description": "A conformance claim disagrees with the active profile identifier or status." + }, + { + "code": "SUITE_DIAGNOSTIC_DUPLICATE", + "phase": "corpus", + "severity": "error", + "description": "The diagnostic registry contains a duplicate diagnostic code." + }, + { + "code": "SUITE_DIAGNOSTIC_UNREGISTERED", + "phase": "corpus", + "severity": "error", + "description": "A case expects or emits a diagnostic absent from the registry." + }, + { + "code": "SUITE_FAMILY_METADATA_MISMATCH", + "phase": "corpus", + "severity": "error", + "description": "Manifest family metadata disagrees with the referenced family document." + }, + { + "code": "SUITE_FILE_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "A suite artifact could not be read." + }, + { + "code": "SUITE_FORMAT_UNSUPPORTED", + "phase": "corpus", + "severity": "error", + "description": "The suite manifest uses an unsupported format version." + }, + { + "code": "SUITE_IMPLEMENTATION_LOCK_INVALID", + "phase": "corpus", + "severity": "error", + "description": "The pinned implementation lock is malformed or uses an unsupported format." + }, + { + "code": "SUITE_JSON_INVALID", + "phase": "corpus", + "severity": "error", + "description": "A suite artifact is not valid JSON or does not match its typed model." + }, + { + "code": "SUITE_PATH_INVALID", + "phase": "io", + "severity": "error", + "description": "A suite artifact path is unsafe or resolves outside the repository." + }, + { + "code": "SUITE_PRODUCER_TRANSCRIPT_REQUIRED", + "phase": "producer", + "severity": "error", + "description": "A Trace producer case does not supply an independent deterministic transcript." + }, + { + "code": "SUITE_PROFILE_COVERAGE_MISSING", + "phase": "corpus", + "severity": "error", + "description": "An active profile requirement has no passing conformance case." + }, + { + "code": "SUITE_PROFILE_REQUIREMENT_UNKNOWN", + "phase": "corpus", + "severity": "error", + "description": "The active profile cites a requirement absent from the inventory." + }, + { + "code": "SUITE_REPORT_INVALID", + "phase": "corpus", + "severity": "error", + "description": "The generated deterministic report does not satisfy the report contract." + }, + { + "code": "TRACE_CONTENT_DIGEST_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "Materialized or supplied content bytes do not match their declared SHA-256 digest." + }, + { + "code": "TRACE_CONTENT_LENGTH_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "Materialized or supplied content does not match its declared UTF-8 byte length." + }, + { + "code": "TRACE_ENTITY_FALLBACK_STEP_INVALID", + "phase": "semantic", + "severity": "error", + "description": "An entity fallback does not identify the first successful authored route step." + }, + { + "code": "TRACE_ENTITY_KEPT_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "An entity marked kept has different requested and applied canonical identifiers." + }, + { + "code": "TRACE_ENTITY_POLICY_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "An entity fill policy disagrees with the routing policy." + }, + { + "code": "TRACE_ENTITY_REFUSAL_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "An entity refusal disagrees with the routing outcome or reports an applied entity." + }, + { + "code": "TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED", + "phase": "semantic", + "severity": "error", + "description": "An entity substitution lacks explicit best-effort policy or the required authority." + }, + { + "code": "TRACE_EVENT_ID_DUPLICATE", + "phase": "semantic", + "severity": "error", + "description": "A turn contains the same event identifier more than once." + }, + { + "code": "TRACE_INLINE_CONTENT_EXCEEDS_THRESHOLD", + "phase": "semantic", + "severity": "error", + "description": "Inline content exceeds the trace's declared UTF-8 byte threshold." + }, + { + "code": "TRACE_PRODUCER_ARTIFACT_DUPLICATE", + "phase": "producer", + "severity": "error", + "description": "A producer transcript supplies the same external artifact digest more than once." + }, + { + "code": "TRACE_PRODUCER_BOUNDARY_MISMATCH", + "phase": "producer", + "severity": "error", + "description": "A boundary outcome differs from the independently observed producer outcome." + }, + { + "code": "TRACE_PRODUCER_BOUNDARY_MISSING", + "phase": "producer", + "severity": "error", + "description": "A trace boundary event has no corresponding independent producer fact." + }, + { + "code": "TRACE_PRODUCER_BOUND_MISMATCH", + "phase": "producer", + "severity": "error", + "description": "The trace's bound entity differs from the entity independently observed as invoked." + }, + { + "code": "TRACE_PRODUCER_FORMAT_UNSUPPORTED", + "phase": "producer", + "severity": "error", + "description": "The deterministic producer transcript uses an unsupported format." + }, + { + "code": "TRACE_PRODUCER_INFERENCE_MISSING", + "phase": "producer", + "severity": "error", + "description": "A trace inference has no corresponding independent producer fact." + }, + { + "code": "TRACE_PRODUCER_ROUTING_OUTCOME_MISMATCH", + "phase": "producer", + "severity": "error", + "description": "The trace routing outcome differs from the independent producer outcome." + }, + { + "code": "TRACE_PRODUCER_TRACE_EVENT_MISSING", + "phase": "producer", + "severity": "error", + "description": "An independently observed event is absent from the trace." + }, + { + "code": "TRACE_PRODUCER_TURN_MISMATCH", + "phase": "producer", + "severity": "error", + "description": "The trace and deterministic producer transcript identify different turns." + }, + { + "code": "TRACE_R1_CONTENT_UNAVAILABLE", + "phase": "producer", + "severity": "error", + "description": "R1 producer validation cannot reconstruct every request byte from supplied evidence." + }, + { + "code": "TRACE_R1_REQUEST_BYTES_MISMATCH", + "phase": "producer", + "severity": "error", + "description": "Reconstructed trace segments differ byte-for-byte from the independent UTF-8 request." + }, + { + "code": "TRACE_R1_SEGMENT_INDEX_INVALID", + "phase": "semantic", + "severity": "error", + "description": "A segment index differs from its zero-based position in the inference." + }, + { + "code": "TRACE_R2_STEP_NOT_FOUND", + "phase": "semantic", + "severity": "error", + "description": "A segment provenance pointer does not resolve within its inference." + }, + { + "code": "TRACE_R3_PARENT_NOT_FOUND", + "phase": "semantic", + "severity": "error", + "description": "A provenance parent pointer does not resolve within its inference." + }, + { + "code": "TRACE_R3_STEP_CYCLE", + "phase": "semantic", + "severity": "error", + "description": "A provenance parent chain contains a cycle." + }, + { + "code": "TRACE_R5_MATERIALIZATION_REQUIRED", + "phase": "semantic", + "severity": "error", + "description": "A materialized oversight export contains unresolved external or withheld content." + }, + { + "code": "TRACE_R5_MODE_TIER_INVALID", + "phase": "semantic", + "severity": "error", + "description": "Materialized content mode is declared outside an oversight-tier export." + }, + { + "code": "TRACE_RESOLUTION_BINDING_UNPINNED", + "phase": "semantic", + "severity": "error", + "description": "A resolution binding is not a canonical pinned entity identifier." + }, + { + "code": "TRACE_RESOLUTION_BOUND_MISSING", + "phase": "semantic", + "severity": "error", + "description": "The resolution report omits the successfully invoked entity." + }, + { + "code": "TRACE_RESOLUTION_REFUSAL_MISSING", + "phase": "semantic", + "severity": "error", + "description": "A refused routing outcome has no typed resolution refusal." + }, + { + "code": "TRACE_ROUTING_ATTEMPT_INVALID", + "phase": "semantic", + "severity": "error", + "description": "Routing outcome and successful attempt count disagree." + }, + { + "code": "TRACE_ROUTING_BOUND_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "The routing bound entity differs from the successful attempt." + }, + { + "code": "TRACE_ROUTING_FILL_MISMATCH", + "phase": "semantic", + "severity": "error", + "description": "The entity fill differs from the routing bound entity." + }, + { + "code": "TRACE_SEGMENT_ID_DUPLICATE", + "phase": "semantic", + "severity": "error", + "description": "A segment identifier occurs more than once within an inference." + }, + { + "code": "TRACE_STEP_ID_DUPLICATE", + "phase": "semantic", + "severity": "error", + "description": "A provenance step identifier occurs more than once within an inference." } ] } diff --git a/conformance/families/trace-boundary-tamper.json b/conformance/families/trace-boundary-tamper.json new file mode 100644 index 0000000..b0dd302 --- /dev/null +++ b/conformance/families/trace-boundary-tamper.json @@ -0,0 +1,160 @@ +{ + "format_version": "0.1-draft", + "id": "trace-boundary-tamper", + "version": "0.1.0-rc.1", + "status": "candidate", + "profile": "trace-user-0-3-draft", + "target": "trace-document", + "cases": [ + { + "id": "boundary-failure-recorded-valid", + "description": "A failed tool call records what was sent, a typed reason, fill, and measurement without inventing received content.", + "requirements": ["PS-TRACE-BOUNDARY-001", "PS-TRACE-EVENTS-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "add", + "path": "/events/-", + "value": { + "id": "boundary-001", + "event": "boundary", + "tool": "tool:example/search@1", + "outcome": "failed", + "sent": { "state": "inline", "text": "quarterly results" }, + "reason": "ENTITY_UNAVAILABLE", + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "status": "refused", + "reason": "ENTITY_UNAVAILABLE", + "deciding_authority": "venue-operations" + } + ], + "measured": { "wall_time_ms": 5 }, + "internal": "outside-trace-perimeter" + } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "boundary-success-received-missing", + "description": "A successful tool call cannot omit what crossed back over the boundary.", + "requirements": ["PS-TRACE-BOUNDARY-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "add", + "path": "/events/-", + "value": { + "id": "boundary-001", + "event": "boundary", + "tool": "tool:example/search@1", + "outcome": "succeeded", + "sent": { "state": "inline", "text": "quarterly results" }, + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "applied": "tool:example/search@1", + "status": "applied" + } + ], + "measured": { "wall_time_ms": 18 } + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["SCHEMA_INSTANCE_INVALID"] + } + }, + { + "id": "tamper-unknown-field-rejected", + "description": "An undeclared field is rejected outside a namespaced extensions object.", + "requirements": ["PS-TRACE-R6"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [{ "op": "add", "path": "/events/0/undisclosed_override", "value": true }], + "expected": { + "conformant": false, + "diagnostics": ["SCHEMA_INSTANCE_INVALID"] + } + }, + { + "id": "tamper-external-digest-mismatch", + "description": "Materialized external content must match its declared SHA-256 digest.", + "requirements": ["PS-TRACE-CONTENT-001", "PS-TRACE-R1"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/segments/0/content", + "value": { + "state": "external", + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "handle": "https://artifacts.example/content/abc", + "materialized": { + "text": "abd", + "materialized_at": "2026-08-03T00:00:00Z" + } + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_CONTENT_DIGEST_MISMATCH"] + } + }, + { + "id": "tamper-external-length-mismatch", + "description": "Materialized external content must match its declared UTF-8 byte length.", + "requirements": ["PS-TRACE-CONTENT-001", "PS-TRACE-R1"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/segments/0/content", + "value": { + "state": "external", + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 4, + "handle": "https://artifacts.example/content/abc", + "materialized": { + "text": "abc", + "materialized_at": "2026-08-03T00:00:00Z" + } + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_CONTENT_LENGTH_MISMATCH"] + } + }, + { + "id": "tamper-duplicate-segment-id", + "description": "Stable segment identifiers cannot be reused within one inference.", + "requirements": ["PS-TRACE-SEGMENT-ID-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "add", + "path": "/events/0/segments/-", + "value": { + "i": 1, + "id": "segment-user-001", + "origin": "user-authored", + "content": { "state": "inline", "text": " Duplicate." }, + "step": "step-user-001" + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_SEGMENT_ID_DUPLICATE"] + } + } + ] +} diff --git a/conformance/families/trace-outcomes.json b/conformance/families/trace-outcomes.json new file mode 100644 index 0000000..064d56a --- /dev/null +++ b/conformance/families/trace-outcomes.json @@ -0,0 +1,319 @@ +{ + "format_version": "0.1-draft", + "id": "trace-outcomes", + "version": "0.1.0-rc.1", + "status": "candidate", + "profile": "trace-user-0-3-draft", + "target": "trace-document", + "cases": [ + { + "id": "entity-kept-strict-valid", + "description": "A strict exact entity is kept and every requested parameter is reported.", + "requirements": ["PS-EXEC-STRICT-001", "PS-CAP-PARAM-001", "PS-TRACE-FILL-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "entity-kept-applied-mismatch", + "description": "A kept outcome cannot name a different applied entity.", + "requirements": ["PS-CORE-BIND-002", "PS-EXEC-STRICT-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing/fill/0/applied", + "value": "model:example/atlas-mini@2026-08-01" + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_ENTITY_KEPT_MISMATCH", "TRACE_ROUTING_FILL_MISMATCH"] + } + }, + { + "id": "entity-substituted-under-strict", + "description": "A strict exact entity request cannot be silently substituted.", + "requirements": ["PS-EXEC-STRICT-001", "PS-TRACE-FILL-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing", + "value": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "outcome": "invoked", + "bound": "model:example/atlas-mini@2026-08-01", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.02, "wall_time_ms": 1200 } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "applied": "model:example/atlas-mini@2026-08-01", + "status": "substituted", + "policy": "strict", + "reason": "ENTITY_SUBSTITUTED", + "deciding_authority": "venue-operations" + } + ] + } + }, + { + "op": "replace", + "path": "/events/0/resolution/bindings/0/bound", + "value": "model:example/atlas-mini@2026-08-01" + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED"] + } + }, + { + "id": "entity-substituted-best-effort-valid", + "description": "An explicit best-effort request may disclose a venue-operations substitution.", + "requirements": ["PS-EXEC-BEST-001", "PS-TRACE-FILL-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing", + "value": { + "requested": "@model:example/atlas-4@2026-08-01", + "policy": "best-effort", + "outcome": "invoked", + "bound": "model:example/atlas-mini@2026-08-01", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.02, "wall_time_ms": 1200 } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "applied": "model:example/atlas-mini@2026-08-01", + "status": "substituted", + "policy": "best-effort", + "reason": "ENTITY_SUBSTITUTED", + "deciding_authority": "venue-operations" + } + ] + } + }, + { + "op": "replace", + "path": "/events/0/resolution/bindings/0/bound", + "value": "model:example/atlas-mini@2026-08-01" + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "entity-authored-fallback-valid", + "description": "A fill failure advances to the first successful authored fallback step.", + "requirements": [ + "PS-EXEC-FALLBACK-001", + "PS-EXEC-FALLBACK-002", + "PS-EXEC-FALLBACK-003", + "PS-TRACE-FILL-001" + ], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing", + "value": { + "requested": "@model:example/atlas-4@2026-08-01! else @model:example/atlas-mini@2026-08-01!", + "policy": "strict", + "outcome": "invoked", + "bound": "model:example/atlas-mini@2026-08-01", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "outcome": "failed", + "reasons": ["ENTITY_UNAVAILABLE"], + "measured": { "wall_time_ms": 20 } + }, + { + "ref": "@model:example/atlas-mini@2026-08-01!", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.02, "wall_time_ms": 1200 } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "applied": "model:example/atlas-mini@2026-08-01", + "status": "fallback", + "policy": "strict", + "route_step": 1 + } + ] + } + }, + { + "op": "replace", + "path": "/events/0/resolution/bindings", + "value": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + }, + { + "ref": "@model:example/atlas-mini@2026-08-01", + "bound": "model:example/atlas-mini@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ] + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "entity-fallback-step-does-not-match", + "description": "The fallback route_step must identify the successful attempt.", + "requirements": ["PS-EXEC-FALLBACK-002", "PS-EXEC-FALLBACK-003"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing", + "value": { + "requested": "@model:example/atlas-4@2026-08-01! else @model:example/atlas-mini@2026-08-01!", + "policy": "strict", + "outcome": "invoked", + "bound": "model:example/atlas-mini@2026-08-01", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "outcome": "failed", + "reasons": ["ENTITY_UNAVAILABLE"], + "measured": { "wall_time_ms": 20 } + }, + { + "ref": "@model:example/atlas-mini@2026-08-01!", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.02, "wall_time_ms": 1200 } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "applied": "model:example/atlas-mini@2026-08-01", + "status": "fallback", + "policy": "strict", + "route_step": 0 + } + ] + } + }, + { + "op": "replace", + "path": "/events/0/resolution/bindings/0/bound", + "value": "model:example/atlas-mini@2026-08-01" + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_ENTITY_FALLBACK_STEP_INVALID"] + } + }, + { + "id": "entity-strict-refusal-valid", + "description": "An unavailable strict exact entity is refused before invocation with recourse.", + "requirements": ["PS-EXEC-STRICT-001", "PS-TRACE-FILL-001", "PS-TRACE-RESOLUTION-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing", + "value": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "outcome": "refused", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "outcome": "failed", + "reasons": ["ENTITY_UNAVAILABLE"], + "measured": { "wall_time_ms": 4 } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "status": "refused", + "policy": "strict", + "reason": "ENTITY_UNAVAILABLE", + "deciding_authority": "venue-operations" + } + ] + } + }, + { + "op": "replace", + "path": "/events/0/resolution", + "value": { + "bindings": [], + "refusals": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "reason": "ENTITY_UNAVAILABLE", + "authority": "venue-operations", + "recourse": { "available": true, "kind": "retry" } + } + ] + } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "routing-bound-disagrees-with-fill", + "description": "The routing bound entity must match the successful attempt and entity fill.", + "requirements": ["PS-CORE-BIND-002", "PS-TRACE-ROUTING-001", "PS-TRACE-FILL-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/routing/bound", + "value": "model:example/atlas-mini@2026-08-01" + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_ROUTING_BOUND_MISMATCH", "TRACE_ROUTING_FILL_MISMATCH"] + } + }, + { + "id": "resolution-missing-applied-binding", + "description": "An invoked canonical entity must appear in the resolution report.", + "requirements": ["PS-TRACE-RESOLUTION-001", "PS-CORE-BIND-002"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [{ "op": "replace", "path": "/events/0/resolution/bindings", "value": [] }], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_RESOLUTION_BOUND_MISSING"] + } + } + ] +} diff --git a/conformance/families/trace-producer-replay.json b/conformance/families/trace-producer-replay.json new file mode 100644 index 0000000..4797bf2 --- /dev/null +++ b/conformance/families/trace-producer-replay.json @@ -0,0 +1,214 @@ +{ + "format_version": "0.1-draft", + "id": "trace-producer-replay", + "version": "0.1.0-rc.1", + "status": "candidate", + "profile": "trace-user-0-3-draft", + "target": "trace-producer", + "cases": [ + { + "id": "producer-request-and-bound-match", + "description": "The trace agrees with independent UTF-8 request bytes and the entity actually invoked.", + "requirements": ["PS-TRACE-R1", "PS-TRACE-ROUTING-001", "PS-CORE-BIND-002"], + "trace": "conformance/fixtures/trace-user-kept.json", + "transcript": "conformance/fixtures/transcript-user-kept.json", + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-request-bytes-mismatch", + "description": "R1 fails when the independent request bytes differ from reconstructed trace segments.", + "requirements": ["PS-TRACE-R1"], + "trace": "conformance/fixtures/trace-user-kept.json", + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [ + { + "op": "replace", + "path": "/inferences/0/actual_request_utf8", + "value": "Summarize the report.\n" + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_R1_REQUEST_BYTES_MISMATCH"] + } + }, + { + "id": "producer-bound-entity-mismatch", + "description": "The trace cannot report Atlas-4 when the deterministic transcript says Atlas-Mini ran.", + "requirements": ["PS-TRACE-ROUTING-001", "PS-CORE-BIND-002"], + "trace": "conformance/fixtures/trace-user-kept.json", + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [ + { + "op": "replace", + "path": "/inferences/0/routing/bound", + "value": "model:example/atlas-mini@2026-08-01" + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_PRODUCER_BOUND_MISMATCH"] + } + }, + { + "id": "producer-turn-mismatch", + "description": "A producer transcript must describe the same turn as the supplied trace.", + "requirements": ["PS-TRACE-EVENTS-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [{ "op": "replace", "path": "/turn", "value": "turn-other-002" }], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_PRODUCER_TURN_MISMATCH"] + } + }, + { + "id": "producer-external-artifact-valid", + "description": "R1 reconstructs external content from independently supplied artifact bytes.", + "requirements": ["PS-TRACE-R1", "PS-TRACE-CONTENT-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/segments/0/content", + "value": { + "state": "external", + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "handle": "https://artifacts.example/content/abc" + } + } + ], + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [ + { "op": "replace", "path": "/inferences/0/actual_request_utf8", "value": "abc" }, + { + "op": "add", + "path": "/external_artifacts/-", + "value": { + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "content_utf8": "abc" + } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-external-artifact-digest-tampered", + "description": "A supplied external artifact is rejected when its bytes do not match its digest.", + "requirements": ["PS-TRACE-R1", "PS-TRACE-CONTENT-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/segments/0/content", + "value": { + "state": "external", + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "handle": "https://artifacts.example/content/abc" + } + } + ], + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [ + { "op": "replace", "path": "/inferences/0/actual_request_utf8", "value": "abc" }, + { + "op": "add", + "path": "/external_artifacts/-", + "value": { + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "content_utf8": "abd" + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_CONTENT_DIGEST_MISMATCH"] + } + }, + { + "id": "producer-boundary-outcome-match", + "description": "A boundary event agrees with the independently observed tool outcome.", + "requirements": ["PS-TRACE-BOUNDARY-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "add", + "path": "/events/-", + "value": { + "id": "boundary-001", + "event": "boundary", + "tool": "tool:example/search@1", + "outcome": "failed", + "sent": { "state": "inline", "text": "query" }, + "reason": "ENTITY_UNAVAILABLE", + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "status": "refused", + "reason": "ENTITY_UNAVAILABLE", + "deciding_authority": "venue-operations" + } + ], + "measured": { "wall_time_ms": 5 } + } + } + ], + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [ + { + "op": "add", + "path": "/boundaries/-", + "value": { "id": "boundary-001", "outcome": "failed" } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-boundary-outcome-mismatch", + "description": "A trace cannot call a tool outcome failed when the independent transcript observed success.", + "requirements": ["PS-TRACE-BOUNDARY-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "add", + "path": "/events/-", + "value": { + "id": "boundary-001", + "event": "boundary", + "tool": "tool:example/search@1", + "outcome": "failed", + "sent": { "state": "inline", "text": "query" }, + "reason": "ENTITY_UNAVAILABLE", + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "status": "refused", + "reason": "ENTITY_UNAVAILABLE", + "deciding_authority": "venue-operations" + } + ], + "measured": { "wall_time_ms": 5 } + } + } + ], + "transcript": "conformance/fixtures/transcript-user-kept.json", + "transcript_patch": [ + { + "op": "add", + "path": "/boundaries/-", + "value": { "id": "boundary-001", "outcome": "succeeded" } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_PRODUCER_BOUNDARY_MISMATCH"] + } + } + ] +} diff --git a/conformance/families/trace-structure.json b/conformance/families/trace-structure.json new file mode 100644 index 0000000..cfef50b --- /dev/null +++ b/conformance/families/trace-structure.json @@ -0,0 +1,248 @@ +{ + "format_version": "0.1-draft", + "id": "trace-structure", + "version": "0.1.0-rc.1", + "status": "candidate", + "profile": "trace-user-0-3-draft", + "target": "trace-document", + "cases": [ + { + "id": "r1-inline-segments-contiguous", + "description": "A user-initiated inline segment list is ordered and reconstructable.", + "requirements": ["PS-TRACE-R1", "PS-TRACE-ASSEMBLY-001", "PS-TRACE-CONTENT-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "r1-segment-index-gap", + "description": "A segment index must equal its zero-based position.", + "requirements": ["PS-TRACE-R1"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [{ "op": "replace", "path": "/events/0/segments/0/i", "value": 1 }], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_R1_SEGMENT_INDEX_INVALID"] + } + }, + { + "id": "r2-r3-injected-provenance-valid", + "description": "An injected segment resolves to an acyclic step chain in its inference.", + "requirements": ["PS-TRACE-R2", "PS-TRACE-R3"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/segments", + "value": [ + { + "i": 0, + "id": "segment-user-001", + "origin": "user-authored", + "content": { "state": "inline", "text": "Summarize " }, + "step": "step-user-001" + }, + { + "i": 1, + "id": "segment-context-001", + "origin": "injected", + "content": { "state": "inline", "text": "the report." }, + "step": "step-context-001" + } + ] + }, + { + "op": "replace", + "path": "/events/0/steps", + "value": [ + { "id": "step-user-001", "kind": "user-content", "tier": "user" }, + { + "id": "step-context-001", + "kind": "context-inclusion", + "parent": "step-user-001", + "tier": "user" + } + ] + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "r2-dangling-segment-step", + "description": "An injected segment cannot point to a missing provenance step.", + "requirements": ["PS-TRACE-R2"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { "op": "replace", "path": "/events/0/segments/0/origin", "value": "injected" }, + { "op": "replace", "path": "/events/0/segments/0/step", "value": "step-missing" } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_R2_STEP_NOT_FOUND"] + } + }, + { + "id": "r3-provenance-cycle", + "description": "A provenance parent chain cannot contain a cycle.", + "requirements": ["PS-TRACE-R3"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/steps", + "value": [ + { + "id": "step-user-001", + "kind": "user-content", + "parent": "step-policy-001", + "tier": "user" + }, + { + "id": "step-policy-001", + "kind": "policy-injection", + "parent": "step-user-001", + "tier": "operator" + } + ] + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_R3_STEP_CYCLE"] + } + }, + { + "id": "r4-user-inline-floor", + "description": "The declared user-tier inline threshold cannot be below 4096 bytes.", + "requirements": ["PS-TRACE-R4"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [{ "op": "replace", "path": "/coverage/inline_threshold_bytes", "value": 1024 }], + "expected": { + "conformant": false, + "diagnostics": ["SCHEMA_INSTANCE_INVALID"] + } + }, + { + "id": "r5-oversight-materialized-valid", + "description": "A materialized oversight export embeds the external content it references.", + "requirements": ["PS-TRACE-R5", "PS-TRACE-CONTENT-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { "op": "replace", "path": "/tier", "value": "oversight" }, + { "op": "replace", "path": "/content_mode", "value": "materialized" }, + { + "op": "replace", + "path": "/events/0/segments/0/content", + "value": { + "state": "external", + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "handle": "https://artifacts.example/content/abc", + "materialized": { + "text": "abc", + "materialized_at": "2026-08-03T00:00:00Z" + } + } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "r5-oversight-materialization-missing", + "description": "A materialized oversight export cannot leave external content unresolved.", + "requirements": ["PS-TRACE-R5"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { "op": "replace", "path": "/tier", "value": "oversight" }, + { "op": "replace", "path": "/content_mode", "value": "materialized" }, + { + "op": "replace", + "path": "/events/0/segments/0/content", + "value": { + "state": "external", + "sha256": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "length": 3, + "handle": "https://artifacts.example/content/abc" + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_R5_MATERIALIZATION_REQUIRED"] + } + }, + { + "id": "r6-internal-omission-declared", + "description": "An internal continuation may omit assembly only through a typed coverage declaration.", + "requirements": ["PS-TRACE-R6", "PS-TRACE-ASSEMBLY-002"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { "op": "replace", "path": "/events/0/kind", "value": "internal-continuation" }, + { + "op": "replace", + "path": "/events/0/segments", + "value": { + "coverage": "not-provided", + "reason": "should-level-not-implemented", + "declared_in": "coverage.assembly" + } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, + { + "id": "r6-user-assembly-cannot-be-omitted", + "description": "A user-initiated inference cannot replace required segments with an omission marker.", + "requirements": ["PS-TRACE-R6", "PS-TRACE-ASSEMBLY-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "replace", + "path": "/events/0/segments", + "value": { + "coverage": "not-provided", + "reason": "should-level-not-implemented", + "declared_in": "coverage.assembly" + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["SCHEMA_INSTANCE_INVALID"] + } + }, + { + "id": "turn-event-identifiers-unique", + "description": "Event identifiers are unique within one turn.", + "requirements": ["PS-TRACE-EVENTS-001"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { + "op": "add", + "path": "/events/-", + "value": { + "id": "inference-001", + "event": "boundary", + "tool": "tool:example/search@1", + "outcome": "failed", + "sent": { "state": "inline", "text": "query" }, + "reason": "ENTITY_UNAVAILABLE", + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "status": "refused", + "reason": "ENTITY_UNAVAILABLE", + "deciding_authority": "venue-operations" + } + ], + "measured": { "wall_time_ms": 5 } + } + } + ], + "expected": { + "conformant": false, + "diagnostics": ["TRACE_EVENT_ID_DUPLICATE"] + } + } + ] +} diff --git a/conformance/fixtures/trace-user-kept.json b/conformance/fixtures/trace-user-kept.json new file mode 100644 index 0000000..d354fa9 --- /dev/null +++ b/conformance/fixtures/trace-user-kept.json @@ -0,0 +1,88 @@ +{ + "ps_trace": "0.3-draft", + "turn": "turn-example-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "vendor_profile": "vendor-profile/example-2026-08" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "events": [ + { + "id": "inference-001", + "event": "inference", + "kind": "user-initiated", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "outcome": "invoked", + "bound": "model:example/atlas-4@2026-08-01", + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { + "cost_usd": 0.041, + "wall_time_ms": 3210 + } + } + ], + "fill": [ + { + "kind": "entity", + "requested": "model:example/atlas-4@2026-08-01", + "applied": "model:example/atlas-4@2026-08-01", + "status": "kept", + "policy": "strict" + }, + { + "kind": "parameter", + "name": "temperature", + "requested": 0.2, + "applied": 0.2, + "status": "applied", + "policy": "strict" + } + ] + }, + "segments": [ + { + "i": 0, + "id": "segment-user-001", + "origin": "user-authored", + "content": { + "state": "inline", + "text": "Summarize the report." + }, + "step": "step-user-001" + } + ], + "steps": [ + { + "id": "step-user-001", + "kind": "user-content", + "tier": "user" + } + ], + "resolution": { + "bindings": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "refusals": [] + } + } + ] +} diff --git a/conformance/fixtures/transcript-user-kept.json b/conformance/fixtures/transcript-user-kept.json new file mode 100644 index 0000000..67728f4 --- /dev/null +++ b/conformance/fixtures/transcript-user-kept.json @@ -0,0 +1,16 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-example-001", + "inferences": [ + { + "id": "inference-001", + "actual_request_utf8": "Summarize the report.", + "routing": { + "outcome": "invoked", + "bound": "model:example/atlas-4@2026-08-01" + } + } + ], + "boundaries": [], + "external_artifacts": [] +} diff --git a/conformance/implementations.lock.json b/conformance/implementations.lock.json new file mode 100644 index 0000000..19f3a61 --- /dev/null +++ b/conformance/implementations.lock.json @@ -0,0 +1,21 @@ +{ + "format_version": "0.1-draft", + "implementations": [ + { + "id": "promptsyntax-rs", + "repository": "pathscale/PromptSyntax-rs", + "commit": "c64072f618ffdb53cc389fc40dfe0370c4cf05be", + "adapter": "tests/conformance.rs", + "fixture": "tests/conformance.json", + "command": ["cargo", "test"] + }, + { + "id": "promptsyntax-ts", + "repository": "pathscale/PromptSyntax-ts", + "commit": "b22acfac066c0ea6eec7d042d69040c695ef5422", + "adapter": "test/parser.test.ts", + "fixture": "test/conformance.json", + "command": ["bun", "test"] + } + ] +} diff --git a/conformance/implementations.schema.json b/conformance/implementations.schema.json new file mode 100644 index 0000000..3901232 --- /dev/null +++ b/conformance/implementations.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/implementations-lock/0.1-draft", + "title": "PromptSyntax pinned conformance implementations", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "implementations"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "implementations": { + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "repository", "commit", "adapter", "fixture", "command"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, + "repository": { "type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "adapter": { "type": "string", "minLength": 1 }, + "fixture": { "type": "string", "minLength": 1 }, + "command": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + } + } + } + } +} diff --git a/conformance/manifest.json b/conformance/manifest.json index 984b937..6171d0f 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -2,21 +2,77 @@ "format_version": "0.1-draft", "suite": { "id": "promptsyntax-conformance", - "version": "0.0.0-bootstrap", - "status": "bootstrap" + "version": "0.1.0-rc.1", + "status": "candidate" }, "spec": { "path": "spec/SPEC.md", "commit": "7456634817f4eb68c8909a58b476505e6fd7c063", - "declared_version": "0.2 with settled 0.2.1 and proposed 0.2.2 material", + "declared_version": "0.3 candidate over the PR #4 working draft", "baseline_status": "working-draft" }, "files": { "requirements": "conformance/requirements.json", "requirements_schema": "conformance/requirements.schema.json", "diagnostics": "conformance/diagnostic-registry.json", - "diagnostics_schema": "conformance/diagnostic-registry.schema.json" + "diagnostics_schema": "conformance/diagnostic-registry.schema.json", + "trace_schema": "schemas/prompt-trace-0.3-draft.schema.json", + "family_schema": "conformance/case-family.schema.json", + "transcript_schema": "conformance/transcript.schema.json", + "profile": "conformance/profiles/trace-user-0.3-draft.json", + "profile_schema": "conformance/profile.schema.json", + "report_schema": "conformance/report.schema.json", + "core_cases": "conformance/cases/core-parser.json", + "core_cases_schema": "conformance/core-cases.schema.json", + "adapter_result_schema": "conformance/adapter-result.schema.json", + "implementations_lock": "conformance/implementations.lock.json", + "implementations_schema": "conformance/implementations.schema.json", + "candidate_report": "conformance/reports/0.1.0-rc.1.json" }, - "families": [], - "claims": [] + "families": [ + { + "id": "trace-structure", + "layer": "trace", + "target": "trace-document", + "version": "0.1.0-rc.1", + "path": "conformance/families/trace-structure.json", + "status": "candidate" + }, + { + "id": "trace-outcomes", + "layer": "trace", + "target": "trace-document", + "version": "0.1.0-rc.1", + "path": "conformance/families/trace-outcomes.json", + "status": "candidate" + }, + { + "id": "trace-boundary-tamper", + "layer": "trace", + "target": "trace-document", + "version": "0.1.0-rc.1", + "path": "conformance/families/trace-boundary-tamper.json", + "status": "candidate" + }, + { + "id": "trace-producer-replay", + "layer": "trace", + "target": "trace-producer", + "version": "0.1.0-rc.1", + "path": "conformance/families/trace-producer-replay.json", + "status": "candidate" + } + ], + "claims": [ + { + "profile": "trace-user-0-3-draft", + "families": [ + "trace-structure", + "trace-outcomes", + "trace-boundary-tamper", + "trace-producer-replay" + ], + "status": "candidate" + } + ] } diff --git a/conformance/manifest.schema.json b/conformance/manifest.schema.json index 0262286..ccc1859 100644 --- a/conformance/manifest.schema.json +++ b/conformance/manifest.schema.json @@ -31,12 +31,41 @@ "files": { "type": "object", "additionalProperties": false, - "required": ["requirements", "requirements_schema", "diagnostics", "diagnostics_schema"], + "required": [ + "requirements", + "requirements_schema", + "diagnostics", + "diagnostics_schema", + "trace_schema", + "family_schema", + "transcript_schema", + "profile", + "profile_schema", + "report_schema", + "core_cases", + "core_cases_schema", + "adapter_result_schema", + "implementations_lock", + "implementations_schema", + "candidate_report" + ], "properties": { "requirements": { "type": "string", "minLength": 1 }, "requirements_schema": { "type": "string", "minLength": 1 }, "diagnostics": { "type": "string", "minLength": 1 }, - "diagnostics_schema": { "type": "string", "minLength": 1 } + "diagnostics_schema": { "type": "string", "minLength": 1 }, + "trace_schema": { "type": "string", "minLength": 1 }, + "family_schema": { "type": "string", "minLength": 1 }, + "transcript_schema": { "type": "string", "minLength": 1 }, + "profile": { "type": "string", "minLength": 1 }, + "profile_schema": { "type": "string", "minLength": 1 }, + "report_schema": { "type": "string", "minLength": 1 }, + "core_cases": { "type": "string", "minLength": 1 }, + "core_cases_schema": { "type": "string", "minLength": 1 }, + "adapter_result_schema": { "type": "string", "minLength": 1 }, + "implementations_lock": { "type": "string", "minLength": 1 }, + "implementations_schema": { "type": "string", "minLength": 1 }, + "candidate_report": { "type": "string", "minLength": 1 } } }, "families": { @@ -51,7 +80,7 @@ "target": { "enum": ["core-parser", "trace-document", "trace-producer"] }, "version": { "type": "string", "minLength": 1 }, "path": { "type": "string", "minLength": 1 }, - "status": { "enum": ["draft", "normative"] } + "status": { "enum": ["candidate", "normative"] } } } }, diff --git a/conformance/profile.schema.json b/conformance/profile.schema.json new file mode 100644 index 0000000..9d215b3 --- /dev/null +++ b/conformance/profile.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/profile/0.1-draft", + "title": "PromptSyntax conformance profile", + "type": "object", + "additionalProperties": false, + "required": [ + "format_version", + "id", + "version", + "status", + "claim", + "spec", + "tier", + "coverage", + "decisions", + "requirements" + ], + "properties": { + "format_version": { "const": "0.1-draft" }, + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, + "version": { "type": "string", "minLength": 1 }, + "status": { "enum": ["candidate", "released"] }, + "claim": { "type": "string", "minLength": 1 }, + "spec": { + "type": "object", + "additionalProperties": false, + "required": ["path", "commit", "declared_version", "baseline_status"], + "properties": { + "path": { "type": "string", "minLength": 1 }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "declared_version": { "type": "string", "minLength": 1 }, + "baseline_status": { "enum": ["working-draft", "frozen"] } + } + }, + "tier": { "enum": ["user", "developer", "operator", "oversight"] }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["routing", "assembly", "standing_authoring_surfaces", "oversight"], + "properties": { + "routing": { "const": "complete" }, + "assembly": { "enum": ["user-initiated-only", "complete"] }, + "standing_authoring_surfaces": { "enum": ["excluded", "included"] }, + "oversight": { "enum": ["excluded", "informative", "included"] } + } + }, + "decisions": { + "type": "object", + "additionalProperties": false, + "required": [ + "exact_entity_policy", + "substitution_policy", + "substitution_authority", + "unknown_fields", + "request_encoding", + "unicode_normalization", + "line_endings" + ], + "properties": { + "exact_entity_policy": { "const": "strict-kept-or-refused" }, + "substitution_policy": { "const": "explicit-best-effort-only" }, + "substitution_authority": { "const": "venue-operations" }, + "unknown_fields": { "const": "reject-except-namespaced-extensions" }, + "request_encoding": { "const": "utf-8" }, + "unicode_normalization": { "const": "none" }, + "line_endings": { "const": "preserve" } + } + }, + "requirements": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^PS-[A-Z0-9-]+$" } + } + } +} diff --git a/conformance/profiles/README.md b/conformance/profiles/README.md new file mode 100644 index 0000000..54e85bd --- /dev/null +++ b/conformance/profiles/README.md @@ -0,0 +1,40 @@ +# Candidate Trace profile + +`trace-user-0-3-draft.json` is the frozen candidate scope for suite `0.1.0-rc.1`. It is +pinned to the PR #4 specification commit and is intentionally narrower than full +PromptSyntax conformance. + +## R1 through R6 interpretation + +- **R1:** segment indices are contiguous and ordered. Trace producer conformance also + compares their reconstructed bytes with independently supplied request bytes encoded as + UTF-8. Compilation performs no implicit Unicode normalization or line-ending conversion. + External bytes come from a digest-checked transcript artifact; missing bytes cannot pass + producer conformance. +- **R2:** every segment provenance pointer resolves to a step in the same inference. +- **R3:** every step parent resolves in the same inference and every parent chain is + acyclic. +- **R4:** the user-tier inline threshold is at least 4096 bytes and inline content cannot + exceed the declared threshold. +- **R5:** an oversight trace declaring `content_mode: materialized` embeds every external + value and contains no withheld content. The embedded bytes must match declared digest and + length metadata. +- **R6:** routing coverage is complete, user-initiated assembly is present, internal + omissions are typed, and every claim names a profile and case families. + +## Entity outcomes + +- `kept` means the requested, applied, successfully attempted, and routing-bound canonical + entities agree. +- `fallback` names the first successful step in an explicitly authored route after prior + fill failures. +- `substituted` is permitted only by an explicitly selected best-effort policy and records + `ENTITY_SUBSTITUTED` with `venue-operations` authority. +- `refused` records no applied entity and includes typed reason, authority, and recourse. + +## Exclusions + +The candidate profile excludes proposed standing authoring surfaces, live provider calls, +claims about whether a self-reporting venue is truthful, and released oversight or +integrity-profile conformance. The R5 cases test the materialization rule as an informative +cross-tier requirement, not a complete oversight profile. diff --git a/conformance/profiles/trace-user-0.3-draft.json b/conformance/profiles/trace-user-0.3-draft.json new file mode 100644 index 0000000..771e3ea --- /dev/null +++ b/conformance/profiles/trace-user-0.3-draft.json @@ -0,0 +1,53 @@ +{ + "format_version": "0.1-draft", + "id": "trace-user-0-3-draft", + "version": "0.1.0-rc.1", + "status": "candidate", + "claim": "PS/Trace 0.3-draft (user tier, routing complete, user-initiated assembly)", + "spec": { + "path": "spec/SPEC.md", + "commit": "7456634817f4eb68c8909a58b476505e6fd7c063", + "declared_version": "0.3 candidate over the PR #4 working draft", + "baseline_status": "working-draft" + }, + "tier": "user", + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "standing_authoring_surfaces": "excluded", + "oversight": "informative" + }, + "decisions": { + "exact_entity_policy": "strict-kept-or-refused", + "substitution_policy": "explicit-best-effort-only", + "substitution_authority": "venue-operations", + "unknown_fields": "reject-except-namespaced-extensions", + "request_encoding": "utf-8", + "unicode_normalization": "none", + "line_endings": "preserve" + }, + "requirements": [ + "PS-CORE-BIND-002", + "PS-CAP-PARAM-001", + "PS-EXEC-STRICT-001", + "PS-EXEC-BEST-001", + "PS-EXEC-FALLBACK-001", + "PS-EXEC-FALLBACK-002", + "PS-EXEC-FALLBACK-003", + "PS-TRACE-R1", + "PS-TRACE-R2", + "PS-TRACE-R3", + "PS-TRACE-R4", + "PS-TRACE-R5", + "PS-TRACE-R6", + "PS-TRACE-EVENTS-001", + "PS-TRACE-ROUTING-001", + "PS-TRACE-ASSEMBLY-001", + "PS-TRACE-ASSEMBLY-002", + "PS-TRACE-BOUNDARY-001", + "PS-TRACE-CONTENT-001", + "PS-TRACE-RESOLUTION-001", + "PS-TRACE-FILL-001", + "PS-TRACE-SEGMENT-ID-001" + ] +} diff --git a/conformance/releases/0.1.0-rc.1.md b/conformance/releases/0.1.0-rc.1.md new file mode 100644 index 0000000..efce841 --- /dev/null +++ b/conformance/releases/0.1.0-rc.1.md @@ -0,0 +1,30 @@ +# PromptSyntax conformance suite 0.1.0-rc.1 + +Status: candidate release prepared for review. No release tag or certification claim has +been issued. + +## Included evidence + +- 34 executable cases in four Trace document and producer families. +- Exact expected diagnostics for positive, negative, and tamper cases. +- Complete coverage of the 22 requirements in `trace-user-0-3-draft`. +- Independent producer transcripts for UTF-8 request bytes, invoked entities, boundary + outcomes, and external artifact bytes. +- One canonical Core parser corpus exercised by pinned Rust and TypeScript implementations. +- A deterministic JSON report with no wall-clock fields. + +## Reproduce + +```bash +cargo test --locked --manifest-path conformance/runner/Cargo.toml +cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-suite . +``` + +## Before tagging + +1. Rebase onto the reviewed PR #4 result and run a requirement-level delta audit. +2. Accept or revise the four provisional semantic defaults in decision 13. +3. Require the Trace suite and both pinned Core adapter jobs to pass in CI. +4. Regenerate and review the deterministic report. +5. Change suite, profile, and included family status only if the baseline is frozen. +6. Create the version tag and archive that immutable revision with the paper artifacts. diff --git a/conformance/report.schema.json b/conformance/report.schema.json new file mode 100644 index 0000000..91f73ff --- /dev/null +++ b/conformance/report.schema.json @@ -0,0 +1,131 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/report/0.1-draft", + "title": "PromptSyntax conformance suite report", + "type": "object", + "additionalProperties": false, + "required": [ + "format_version", + "suite", + "spec", + "profile", + "conformant", + "summary", + "coverage", + "families", + "diagnostics" + ], + "properties": { + "format_version": { "const": "0.1-draft" }, + "suite": { "$ref": "#/$defs/versioned_identity" }, + "spec": { + "type": "object", + "additionalProperties": false, + "required": ["commit", "declared_version"], + "properties": { + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "declared_version": { "type": "string", "minLength": 1 } + } + }, + "profile": { "$ref": "#/$defs/versioned_identity" }, + "conformant": { "type": "boolean" }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "families", + "cases", + "passed", + "failed", + "requirements_required", + "requirements_covered" + ], + "properties": { + "families": { "type": "integer", "minimum": 0 }, + "cases": { "type": "integer", "minimum": 0 }, + "passed": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 }, + "requirements_required": { "type": "integer", "minimum": 0 }, + "requirements_covered": { "type": "integer", "minimum": 0 } + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["required", "covered", "missing"], + "properties": { + "required": { "$ref": "#/$defs/string_set" }, + "covered": { "$ref": "#/$defs/string_set" }, + "missing": { "$ref": "#/$defs/string_set" } + } + }, + "families": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "target", "status", "cases"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "target": { "enum": ["trace-document", "trace-producer"] }, + "status": { "enum": ["candidate", "normative"] }, + "cases": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "passed", + "expected_conformant", + "actual_conformant", + "expected_diagnostics", + "actual_diagnostics", + "requirements" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "passed": { "type": "boolean" }, + "expected_conformant": { "type": "boolean" }, + "actual_conformant": { "type": "boolean" }, + "expected_diagnostics": { "$ref": "#/$defs/string_set" }, + "actual_diagnostics": { "$ref": "#/$defs/string_set" }, + "requirements": { "$ref": "#/$defs/string_set" } + } + } + } + } + } + }, + "diagnostics": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "pointer", "message"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "pointer": { "type": "string" }, + "message": { "type": "string", "minLength": 1 } + } + } + } + }, + "$defs": { + "versioned_identity": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "status"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "status": { "type": "string", "minLength": 1 } + } + }, + "string_set": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string" } + } + } +} diff --git a/conformance/reports/0.1.0-rc.1.json b/conformance/reports/0.1.0-rc.1.json new file mode 100644 index 0000000..a9f8b17 --- /dev/null +++ b/conformance/reports/0.1.0-rc.1.json @@ -0,0 +1,602 @@ +{ + "format_version": "0.1-draft", + "suite": { + "id": "promptsyntax-conformance", + "version": "0.1.0-rc.1", + "status": "candidate" + }, + "spec": { + "commit": "7456634817f4eb68c8909a58b476505e6fd7c063", + "declared_version": "0.3 candidate over the PR #4 working draft" + }, + "profile": { + "id": "trace-user-0-3-draft", + "version": "0.1.0-rc.1", + "status": "candidate" + }, + "conformant": true, + "summary": { + "families": 4, + "cases": 34, + "passed": 34, + "failed": 0, + "requirements_required": 22, + "requirements_covered": 22 + }, + "coverage": { + "required": [ + "PS-CAP-PARAM-001", + "PS-CORE-BIND-002", + "PS-EXEC-BEST-001", + "PS-EXEC-FALLBACK-001", + "PS-EXEC-FALLBACK-002", + "PS-EXEC-FALLBACK-003", + "PS-EXEC-STRICT-001", + "PS-TRACE-ASSEMBLY-001", + "PS-TRACE-ASSEMBLY-002", + "PS-TRACE-BOUNDARY-001", + "PS-TRACE-CONTENT-001", + "PS-TRACE-EVENTS-001", + "PS-TRACE-FILL-001", + "PS-TRACE-R1", + "PS-TRACE-R2", + "PS-TRACE-R3", + "PS-TRACE-R4", + "PS-TRACE-R5", + "PS-TRACE-R6", + "PS-TRACE-RESOLUTION-001", + "PS-TRACE-ROUTING-001", + "PS-TRACE-SEGMENT-ID-001" + ], + "covered": [ + "PS-CAP-PARAM-001", + "PS-CORE-BIND-002", + "PS-EXEC-BEST-001", + "PS-EXEC-FALLBACK-001", + "PS-EXEC-FALLBACK-002", + "PS-EXEC-FALLBACK-003", + "PS-EXEC-STRICT-001", + "PS-TRACE-ASSEMBLY-001", + "PS-TRACE-ASSEMBLY-002", + "PS-TRACE-BOUNDARY-001", + "PS-TRACE-CONTENT-001", + "PS-TRACE-EVENTS-001", + "PS-TRACE-FILL-001", + "PS-TRACE-R1", + "PS-TRACE-R2", + "PS-TRACE-R3", + "PS-TRACE-R4", + "PS-TRACE-R5", + "PS-TRACE-R6", + "PS-TRACE-RESOLUTION-001", + "PS-TRACE-ROUTING-001", + "PS-TRACE-SEGMENT-ID-001" + ], + "missing": [] + }, + "families": [ + { + "id": "trace-boundary-tamper", + "target": "trace-document", + "status": "candidate", + "cases": [ + { + "id": "boundary-failure-recorded-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-BOUNDARY-001", + "PS-TRACE-EVENTS-001" + ] + }, + { + "id": "boundary-success-received-missing", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "actual_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "requirements": [ + "PS-TRACE-BOUNDARY-001" + ] + }, + { + "id": "tamper-duplicate-segment-id", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_SEGMENT_ID_DUPLICATE" + ], + "actual_diagnostics": [ + "TRACE_SEGMENT_ID_DUPLICATE" + ], + "requirements": [ + "PS-TRACE-SEGMENT-ID-001" + ] + }, + { + "id": "tamper-external-digest-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ], + "requirements": [ + "PS-TRACE-CONTENT-001", + "PS-TRACE-R1" + ] + }, + { + "id": "tamper-external-length-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_CONTENT_LENGTH_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_CONTENT_LENGTH_MISMATCH" + ], + "requirements": [ + "PS-TRACE-CONTENT-001", + "PS-TRACE-R1" + ] + }, + { + "id": "tamper-unknown-field-rejected", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "actual_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "requirements": [ + "PS-TRACE-R6" + ] + } + ] + }, + { + "id": "trace-outcomes", + "target": "trace-document", + "status": "candidate", + "cases": [ + { + "id": "entity-authored-fallback-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-EXEC-FALLBACK-001", + "PS-EXEC-FALLBACK-002", + "PS-EXEC-FALLBACK-003", + "PS-TRACE-FILL-001" + ] + }, + { + "id": "entity-fallback-step-does-not-match", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_ENTITY_FALLBACK_STEP_INVALID" + ], + "actual_diagnostics": [ + "TRACE_ENTITY_FALLBACK_STEP_INVALID" + ], + "requirements": [ + "PS-EXEC-FALLBACK-002", + "PS-EXEC-FALLBACK-003" + ] + }, + { + "id": "entity-kept-applied-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_ENTITY_KEPT_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_ENTITY_KEPT_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ], + "requirements": [ + "PS-CORE-BIND-002", + "PS-EXEC-STRICT-001" + ] + }, + { + "id": "entity-kept-strict-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-CAP-PARAM-001", + "PS-EXEC-STRICT-001", + "PS-TRACE-FILL-001" + ] + }, + { + "id": "entity-strict-refusal-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-EXEC-STRICT-001", + "PS-TRACE-FILL-001", + "PS-TRACE-RESOLUTION-001" + ] + }, + { + "id": "entity-substituted-best-effort-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-EXEC-BEST-001", + "PS-TRACE-FILL-001" + ] + }, + { + "id": "entity-substituted-under-strict", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED" + ], + "actual_diagnostics": [ + "TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED" + ], + "requirements": [ + "PS-EXEC-STRICT-001", + "PS-TRACE-FILL-001" + ] + }, + { + "id": "resolution-missing-applied-binding", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_RESOLUTION_BOUND_MISSING" + ], + "actual_diagnostics": [ + "TRACE_RESOLUTION_BOUND_MISSING" + ], + "requirements": [ + "PS-CORE-BIND-002", + "PS-TRACE-RESOLUTION-001" + ] + }, + { + "id": "routing-bound-disagrees-with-fill", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_ROUTING_BOUND_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_ROUTING_BOUND_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ], + "requirements": [ + "PS-CORE-BIND-002", + "PS-TRACE-FILL-001", + "PS-TRACE-ROUTING-001" + ] + } + ] + }, + { + "id": "trace-producer-replay", + "target": "trace-producer", + "status": "candidate", + "cases": [ + { + "id": "producer-bound-entity-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_PRODUCER_BOUND_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_PRODUCER_BOUND_MISMATCH" + ], + "requirements": [ + "PS-CORE-BIND-002", + "PS-TRACE-ROUTING-001" + ] + }, + { + "id": "producer-boundary-outcome-match", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-BOUNDARY-001" + ] + }, + { + "id": "producer-boundary-outcome-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_PRODUCER_BOUNDARY_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_PRODUCER_BOUNDARY_MISMATCH" + ], + "requirements": [ + "PS-TRACE-BOUNDARY-001" + ] + }, + { + "id": "producer-external-artifact-digest-tampered", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ], + "requirements": [ + "PS-TRACE-CONTENT-001", + "PS-TRACE-R1" + ] + }, + { + "id": "producer-external-artifact-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-CONTENT-001", + "PS-TRACE-R1" + ] + }, + { + "id": "producer-request-and-bound-match", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-CORE-BIND-002", + "PS-TRACE-R1", + "PS-TRACE-ROUTING-001" + ] + }, + { + "id": "producer-request-bytes-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_R1_REQUEST_BYTES_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_R1_REQUEST_BYTES_MISMATCH" + ], + "requirements": [ + "PS-TRACE-R1" + ] + }, + { + "id": "producer-turn-mismatch", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_PRODUCER_TURN_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_PRODUCER_TURN_MISMATCH" + ], + "requirements": [ + "PS-TRACE-EVENTS-001" + ] + } + ] + }, + { + "id": "trace-structure", + "target": "trace-document", + "status": "candidate", + "cases": [ + { + "id": "r1-inline-segments-contiguous", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-ASSEMBLY-001", + "PS-TRACE-CONTENT-001", + "PS-TRACE-R1" + ] + }, + { + "id": "r1-segment-index-gap", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_R1_SEGMENT_INDEX_INVALID" + ], + "actual_diagnostics": [ + "TRACE_R1_SEGMENT_INDEX_INVALID" + ], + "requirements": [ + "PS-TRACE-R1" + ] + }, + { + "id": "r2-dangling-segment-step", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_R2_STEP_NOT_FOUND" + ], + "actual_diagnostics": [ + "TRACE_R2_STEP_NOT_FOUND" + ], + "requirements": [ + "PS-TRACE-R2" + ] + }, + { + "id": "r2-r3-injected-provenance-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-R2", + "PS-TRACE-R3" + ] + }, + { + "id": "r3-provenance-cycle", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_R3_STEP_CYCLE" + ], + "actual_diagnostics": [ + "TRACE_R3_STEP_CYCLE" + ], + "requirements": [ + "PS-TRACE-R3" + ] + }, + { + "id": "r4-user-inline-floor", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "actual_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "requirements": [ + "PS-TRACE-R4" + ] + }, + { + "id": "r5-oversight-materialization-missing", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_R5_MATERIALIZATION_REQUIRED" + ], + "actual_diagnostics": [ + "TRACE_R5_MATERIALIZATION_REQUIRED" + ], + "requirements": [ + "PS-TRACE-R5" + ] + }, + { + "id": "r5-oversight-materialized-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-CONTENT-001", + "PS-TRACE-R5" + ] + }, + { + "id": "r6-internal-omission-declared", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-ASSEMBLY-002", + "PS-TRACE-R6" + ] + }, + { + "id": "r6-user-assembly-cannot-be-omitted", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "actual_diagnostics": [ + "SCHEMA_INSTANCE_INVALID" + ], + "requirements": [ + "PS-TRACE-ASSEMBLY-001", + "PS-TRACE-R6" + ] + }, + { + "id": "turn-event-identifiers-unique", + "passed": true, + "expected_conformant": false, + "actual_conformant": false, + "expected_diagnostics": [ + "TRACE_EVENT_ID_DUPLICATE" + ], + "actual_diagnostics": [ + "TRACE_EVENT_ID_DUPLICATE" + ], + "requirements": [ + "PS-TRACE-EVENTS-001" + ] + } + ] + } + ], + "diagnostics": [] +} diff --git a/conformance/requirements.json b/conformance/requirements.json index e51548f..dfc70bf 100644 --- a/conformance/requirements.json +++ b/conformance/requirements.json @@ -53,7 +53,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "5.2", "candidate_anchor": "ps-core-span-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "5.2", + "candidate_anchor": "ps-core-span-001" + }, "statement": "A model-routing reference inside a span is rejected rather than ignored and reports SPAN_EXECUTION_UNSUPPORTED.", "testability": "automatic", "targets": ["core-parser", "execution-resolver"] @@ -63,7 +67,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "6.2", "candidate_anchor": "ps-core-bind-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "6.2", + "candidate_anchor": "ps-core-bind-001" + }, "statement": "Resolution binds each reference to a globally canonical, immutable, versioned identifier.", "testability": "automatic", "targets": ["execution-resolver", "trace-document"] @@ -73,7 +81,11 @@ "layer": "core", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "6.2", "candidate_anchor": "ps-core-bind-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "6.2", + "candidate_anchor": "ps-core-bind-002" + }, "statement": "A trace never reports an unpinned entity binding.", "testability": "automatic", "targets": ["trace-document"] @@ -83,7 +95,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "6.3", "candidate_anchor": "ps-core-ambig-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "6.3", + "candidate_anchor": "ps-core-ambig-001" + }, "statement": "Multiple matching entities produce a surfaced ENTITY_AMBIGUOUS result.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -93,7 +109,11 @@ "layer": "core", "normative_status": "settled", "strength": "must_not", - "source": { "path": "spec/SPEC.md", "section": "6.3", "candidate_anchor": "ps-core-ambig-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "6.3", + "candidate_anchor": "ps-core-ambig-002" + }, "statement": "An implementation must not silently choose among multiple matching entities.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -103,7 +123,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "6.4", "candidate_anchor": "ps-core-action-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "6.4", + "candidate_anchor": "ps-core-action-001" + }, "statement": "A consequential action is fully qualified or explicitly confirmed by the user.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -113,7 +137,11 @@ "layer": "core", "normative_status": "settled", "strength": "should", - "source": { "path": "spec/SPEC.md", "section": "6.4", "candidate_anchor": "ps-core-strict-mode-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "6.4", + "candidate_anchor": "ps-core-strict-mode-001" + }, "statement": "A venue should offer a mode in which only ps tags and strict frontmatter are executable.", "testability": "profile", "targets": ["capability-document", "core-parser"] @@ -123,7 +151,11 @@ "layer": "core", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "11", "candidate_anchor": "ps-core-roundtrip-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "11", + "candidate_anchor": "ps-core-roundtrip-001" + }, "statement": "Compiling casual form is equivalent to compiling the strict form produced from that casual compilation.", "testability": "automatic", "targets": ["core-parser", "execution-resolver"] @@ -133,7 +165,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "11", "candidate_anchor": "ps-core-reify-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "11", + "candidate_anchor": "ps-core-reify-001" + }, "statement": "Intent reification emits an editable and re-submittable strict form.", "testability": "automatic", "targets": ["execution-resolver"] @@ -153,7 +189,11 @@ "layer": "core", "normative_status": "settled", "strength": "must_not", - "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-inert-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13", + "candidate_anchor": "ps-core-inert-001" + }, "statement": "Retrieved documents, file contents, tool results, quoted messages, and model-generated text do not introduce directives without authority-checked promotion.", "testability": "automatic", "targets": ["core-parser", "trace-producer"] @@ -163,7 +203,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-serialize-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13", + "candidate_anchor": "ps-core-serialize-001" + }, "statement": "Canonical serialization carries included content by reference or typed data and never splices it into directive text.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -173,7 +217,11 @@ "layer": "core", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-promote-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13", + "candidate_anchor": "ps-core-promote-001" + }, "statement": "Promotion is explicit, authority-checked, and recorded in the trace, and it changes parse eligibility without granting capability.", "testability": "blocked", "targets": ["core-parser", "trace-producer"], @@ -195,7 +243,11 @@ "layer": "core", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-boundary-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13", + "candidate_anchor": "ps-core-boundary-001" + }, "statement": "Point islands require the defined boundary and a qualified or environment-resolvable name; email addresses and paths remain content.", "testability": "automatic", "targets": ["core-parser"] @@ -205,7 +257,11 @@ "layer": "core", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-escape-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13", + "candidate_anchor": "ps-core-escape-001" + }, "statement": "Escaped sigils and fenced code blocks always remain content.", "testability": "automatic", "targets": ["core-parser"] @@ -215,7 +271,11 @@ "layer": "core", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13", "candidate_anchor": "ps-core-highlight-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13", + "candidate_anchor": "ps-core-highlight-001" + }, "statement": "An interactive implementation visibly marks every recognized directive before submission.", "testability": "manual", "targets": ["interactive-ui"] @@ -225,7 +285,11 @@ "layer": "core", "normative_status": "proposed", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-width-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.1", + "candidate_anchor": "ps-core-width-001" + }, "statement": "Parsers treat full-width at and slash sigils as equivalent to their ASCII forms.", "testability": "automatic", "targets": ["core-parser"] @@ -235,7 +299,11 @@ "layer": "core", "normative_status": "proposed", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-unicode-id-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.1", + "candidate_anchor": "ps-core-unicode-id-001" + }, "statement": "Entity identifiers follow UAX 31 with NFC normalization while canonical keywords remain English.", "testability": "automatic", "targets": ["core-parser"] @@ -245,7 +313,11 @@ "layer": "core", "normative_status": "proposed", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-segment-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.1", + "candidate_anchor": "ps-core-segment-001" + }, "statement": "Island boundaries follow UAX 29 and work in scripts without inter-word spaces.", "testability": "automatic", "targets": ["core-parser"] @@ -255,7 +327,11 @@ "layer": "core", "normative_status": "proposed", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-corpus-i18n-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.1", + "candidate_anchor": "ps-core-corpus-i18n-001" + }, "statement": "The conformance corpus includes cases for scripts without inter-word spacing.", "testability": "automatic", "targets": ["corpus"] @@ -265,7 +341,11 @@ "layer": "core", "normative_status": "proposed", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13.1", "candidate_anchor": "ps-core-bidi-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.1", + "candidate_anchor": "ps-core-bidi-001" + }, "statement": "A parser rejects or neutralizes bidi control characters inside directive islands without leaking a partial directive.", "testability": "automatic", "targets": ["core-parser"] @@ -275,7 +355,11 @@ "layer": "core", "normative_status": "proposed", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "13.2", "candidate_anchor": "ps-core-surface-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.2", + "candidate_anchor": "ps-core-surface-001" + }, "statement": "A standing authoring-surface declaration binds identity, lifecycle, origin, frame, action set, capability scope, replay behavior, and trace obligations.", "testability": "blocked", "targets": ["capability-document", "core-parser", "trace-producer"], @@ -286,7 +370,11 @@ "layer": "core", "normative_status": "proposed", "strength": "must_not", - "source": { "path": "spec/SPEC.md", "section": "13.2", "candidate_anchor": "ps-core-surface-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "13.2", + "candidate_anchor": "ps-core-surface-002" + }, "statement": "A host must not search undifferentiated transcript, concatenated prose, quoted content, or retrieved data for a surface delimiter.", "testability": "blocked", "targets": ["core-parser"], @@ -309,7 +397,11 @@ "layer": "capabilities", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "7.1", "candidate_anchor": "ps-cap-version-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "7.1", + "candidate_anchor": "ps-cap-version-001" + }, "statement": "A venue rejects a declared PromptSyntax version it cannot honor instead of degrading silently.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -319,7 +411,11 @@ "layer": "capabilities", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "7.2", "candidate_anchor": "ps-cap-param-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "7.2", + "candidate_anchor": "ps-cap-param-001" + }, "statement": "Every parameter records requested, normalized, and applied values with a defined status in the fill report.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -329,7 +425,11 @@ "layer": "capabilities", "normative_status": "settled", "strength": "must_not", - "source": { "path": "spec/SPEC.md", "section": "7.2", "candidate_anchor": "ps-cap-param-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "7.2", + "candidate_anchor": "ps-cap-param-002" + }, "statement": "An implementation must not claim undocumented parameter passthrough as conforming normalization.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -339,7 +439,11 @@ "layer": "capabilities", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.2", "candidate_anchor": "ps-cap-tier-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.2", + "candidate_anchor": "ps-cap-tier-001" + }, "statement": "The capability document declares the trace tiers supported by the venue.", "testability": "blocked", "targets": ["capability-document", "trace-document"], @@ -351,7 +455,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-strict-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "8.1", + "candidate_anchor": "ps-exec-strict-001" + }, "statement": "Strict fulfillment applies every requirement exactly or fails before execution.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -361,7 +469,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-partial-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "8.1", + "candidate_anchor": "ps-exec-partial-001" + }, "statement": "Partial fulfillment records every dropped requirement.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -371,7 +483,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-best-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "8.1", + "candidate_anchor": "ps-exec-best-001" + }, "statement": "Best-effort fulfillment records every degradation.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -381,7 +497,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-default-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "8.1", + "candidate_anchor": "ps-exec-default-001" + }, "statement": "The default policy is strict entity binding and partial parameter fulfillment.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -391,7 +511,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "8.1", "candidate_anchor": "ps-exec-bang-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "8.1", + "candidate_anchor": "ps-exec-bang-001" + }, "statement": "A bang marker makes the entire directive strict, permitting only declared semantics-preserving translation.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -401,7 +525,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "8.2", "candidate_anchor": "ps-exec-reason-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "8.2", + "candidate_anchor": "ps-exec-reason-001" + }, "statement": "Every fill failure and disclosed degradation carries a standardized or permitted extension reason code.", "testability": "blocked", "targets": ["trace-document", "trace-producer"], @@ -412,7 +540,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.1", + "candidate_anchor": "ps-exec-budget-001" + }, "statement": "A violated budget is a fill failure.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -422,7 +554,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.1", + "candidate_anchor": "ps-exec-budget-002" + }, "statement": "Required cancellation against a venue declaring cancellation unsupported fails before execution with BUDGET_PREFLIGHT_REJECTED.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -432,7 +568,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-003" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.1", + "candidate_anchor": "ps-exec-budget-003" + }, "statement": "A timeout that the venue could not cancel reports TIMEOUT_NOT_CANCELLED with observed cost.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -442,7 +582,11 @@ "layer": "execution", "normative_status": "settled", "strength": "must_not", - "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-004" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.1", + "candidate_anchor": "ps-exec-budget-004" + }, "statement": "An implementation must not silently truncate or otherwise degrade output to satisfy a budget.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -452,7 +596,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.1", "candidate_anchor": "ps-exec-budget-005" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.1", + "candidate_anchor": "ps-exec-budget-005" + }, "statement": "A chain-scoped budget applies to the total across fallback attempts.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -462,7 +610,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.2", + "candidate_anchor": "ps-exec-fallback-001" + }, "statement": "A fill failure is the only trigger for advancing an authored fallback chain.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -472,7 +624,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.2", + "candidate_anchor": "ps-exec-fallback-002" + }, "statement": "The first successful legal fallback step wins.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -482,7 +638,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-003" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.2", + "candidate_anchor": "ps-exec-fallback-003" + }, "statement": "The fill report records every fallback attempt in order with outcome, codes, measured cost, and measured latency.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -492,7 +652,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "9.2", "candidate_anchor": "ps-exec-fallback-004" }, + "source": { + "path": "spec/SPEC.md", + "section": "9.2", + "candidate_anchor": "ps-exec-fallback-004" + }, "statement": "Every fallback step is checked against the same applicable capability, authorization, consent, data, residency, retention, and connector constraints.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -502,7 +666,11 @@ "layer": "execution", "normative_status": "settled", "strength": "must_not", - "source": { "path": "spec/SPEC.md", "section": "10.1", "candidate_anchor": "ps-exec-auth-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "10.1", + "candidate_anchor": "ps-exec-auth-001" + }, "statement": "A lower authority must not expand permissions, budgets, or entity sets granted by higher authority.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -512,7 +680,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "10.2", "candidate_anchor": "ps-exec-specificity-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "10.2", + "candidate_anchor": "ps-exec-specificity-001" + }, "statement": "Within one authority level, span scope overrides point scope and point scope overrides document scope.", "testability": "automatic", "targets": ["execution-resolver", "trace-producer"] @@ -522,7 +694,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "10.3", "candidate_anchor": "ps-exec-envelope-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "10.3", + "candidate_anchor": "ps-exec-envelope-001" + }, "statement": "An out-of-envelope directive fails with ENTITY_UNAUTHORIZED or DATA_POLICY_BLOCKED and is never silently ignored.", "testability": "automatic", "targets": ["execution-resolver", "trace-document", "trace-producer"] @@ -532,7 +708,11 @@ "layer": "execution", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "10.3", "candidate_anchor": "ps-exec-envelope-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "10.3", + "candidate_anchor": "ps-exec-envelope-002" + }, "statement": "Every envelope constraint that suppresses a directive is recorded as a typed refusal at the requesting user's tier.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -542,7 +722,11 @@ "layer": "execution", "normative_status": "proposed", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "10.5", "candidate_anchor": "ps-exec-refusal-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "10.5", + "candidate_anchor": "ps-exec-refusal-001" + }, "statement": "A refusal distinguishes availability, safety, organization or data policy, and authorization, and records authority and available recourse.", "testability": "blocked", "targets": ["trace-document", "trace-producer"], @@ -615,7 +799,11 @@ "layer": "trace", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-events-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-events-001" + }, "statement": "A Prompt Trace is a turn-level object whose ordered event array interleaves inference and boundary events in occurrence order.", "testability": "automatic", "targets": ["trace-document"] @@ -625,7 +813,11 @@ "layer": "trace", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-routing-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-routing-001" + }, "statement": "Every venue invocation records a routing record and fill report.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -635,7 +827,11 @@ "layer": "trace", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-assembly-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-assembly-001" + }, "statement": "Every user-initiated inference records full tier-scoped compiled prompt segments.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -645,7 +841,11 @@ "layer": "trace", "normative_status": "settled", "strength": "should", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-assembly-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-assembly-002" + }, "statement": "An internal continuation should record full tier-scoped compiled prompt segments, and an omission is typed and declared.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -655,7 +855,11 @@ "layer": "trace", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-boundary-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-boundary-001" + }, "statement": "Every tool input and output crossing the venue boundary is recorded while tool internals remain outside the perimeter.", "testability": "blocked", "targets": ["trace-document", "trace-producer"], @@ -666,7 +870,11 @@ "layer": "trace", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-content-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-content-001" + }, "statement": "Trace content uses exactly one tier-appropriate inline, external, or withheld state.", "testability": "automatic", "targets": ["trace-document"] @@ -676,7 +884,11 @@ "layer": "trace", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-resolution-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-resolution-001" + }, "statement": "A trace records every binding, deciding rule, surfaced ambiguity, and envelope refusal in its resolution report.", "testability": "automatic", "targets": ["trace-document", "trace-producer"] @@ -686,7 +898,11 @@ "layer": "trace", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-fill-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-fill-001" + }, "statement": "A routing record includes outcome codes, normalization records, and per-attempt outcomes with measured cost and latency.", "testability": "blocked", "targets": ["trace-document", "trace-producer"], @@ -697,7 +913,11 @@ "layer": "trace", "normative_status": "settled", "strength": "should", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-dryrun-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-dryrun-001" + }, "statement": "An implementation should support a pre-execution dry-run trace.", "testability": "profile", "targets": ["capability-document", "trace-producer"] @@ -707,7 +927,11 @@ "layer": "trace", "normative_status": "settled", "strength": "should", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-segment-id-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-segment-id-001" + }, "statement": "An implementation should produce stable segment identifiers suitable for trace diffing.", "testability": "automatic", "targets": ["trace-producer"] @@ -717,7 +941,11 @@ "layer": "trace", "normative_status": "settled", "strength": "may", - "source": { "path": "spec/SPEC.md", "section": "12.1", "candidate_anchor": "ps-trace-presentation-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.1", + "candidate_anchor": "ps-trace-presentation-001" + }, "statement": "Presentation of conforming trace information is implementation-defined.", "testability": "manual", "targets": ["documentation", "interactive-ui"] @@ -727,7 +955,11 @@ "layer": "trace", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.2", "candidate_anchor": "ps-trace-tier-user-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.2", + "candidate_anchor": "ps-trace-tier-user-001" + }, "statement": "A user-tier trace exposes user-visible sources, selected tools, applied parameters, typed envelope refusals, and the fill report.", "testability": "blocked", "targets": ["trace-document", "trace-producer"], @@ -738,7 +970,11 @@ "layer": "trace", "normative_status": "open", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.2", "candidate_anchor": "ps-trace-placeholder-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.2", + "candidate_anchor": "ps-trace-placeholder-001" + }, "statement": "Content above the current tier appears as a typed and safely sized placeholder.", "testability": "blocked", "targets": ["trace-document", "trace-producer"], @@ -749,7 +985,11 @@ "layer": "trace", "normative_status": "settled", "strength": "should", - "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-integrity-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.3", + "candidate_anchor": "ps-trace-integrity-001" + }, "statement": "Core Trace should support integrity chaining or signing, with stronger requirements defined by oversight profiles.", "testability": "blocked", "targets": ["oversight-profile", "trace-document"], @@ -760,7 +1000,11 @@ "layer": "trace", "normative_status": "settled", "strength": "must", - "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-oversight-001" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.3", + "candidate_anchor": "ps-trace-oversight-001" + }, "statement": "An implementation claiming oversight-tier support records retention and residency metadata.", "testability": "blocked", "targets": ["oversight-profile", "trace-document", "trace-producer"], @@ -771,7 +1015,11 @@ "layer": "trace", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-oversight-002" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.3", + "candidate_anchor": "ps-trace-oversight-002" + }, "statement": "An oversight evidence stream represents omitted events with explicit signed sequence gaps rather than silence.", "testability": "blocked", "targets": ["oversight-profile", "trace-document", "trace-producer"], @@ -782,7 +1030,11 @@ "layer": "trace", "normative_status": "settled", "strength": "invariant", - "source": { "path": "spec/SPEC.md", "section": "12.3", "candidate_anchor": "ps-trace-oversight-003" }, + "source": { + "path": "spec/SPEC.md", + "section": "12.3", + "candidate_anchor": "ps-trace-oversight-003" + }, "statement": "A withheld oversight segment supports evidence of its existence, kind, and timing without revealing its content.", "testability": "blocked", "targets": ["oversight-profile", "trace-document", "trace-producer"], diff --git a/conformance/runner/Cargo.lock b/conformance/runner/Cargo.lock new file mode 100644 index 0000000..54607c0 --- /dev/null +++ b/conformance/runner/Cargo.lock @@ -0,0 +1,1041 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "jsonschema" +version = "0.49.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "508004a5500f2e1f68af048f70feea2de86d35ab115d85716530860822aef397" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom", + "idna", + "itoa", + "jsonschema-regex", + "jsonschema-value", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "strum", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.49.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8b30cafa78358ae6cd1494a7d6410b89530e28bf567f862c869c667e900d9f" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "jsonschema-value" +version = "0.49.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5526bd381d230af94908d07e6835a33fd82a465e12f5f1e9c81f5c2aa23b3c21" +dependencies = [ + "ahash", + "bytecount", + "fraction", + "num-cmp", + "num-traits", + "serde_json", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "promptsyntax-conformance" +version = "0.0.0" +dependencies = [ + "jsonschema", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "referencing" +version = "0.49.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af3eb523cce0df0af3c30d624b829b2dabd233172b5bc2615fcd03ceae8f746" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom", + "hashbrown", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/conformance/runner/Cargo.toml b/conformance/runner/Cargo.toml new file mode 100644 index 0000000..ef02a44 --- /dev/null +++ b/conformance/runner/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "promptsyntax-conformance" +version = "0.0.0" +edition = "2024" +rust-version = "1.85" +description = "Repository-owned conformance runner for PromptSyntax" +license = "MIT" +publish = false + +[[bin]] +name = "ps-conformance" +path = "src/main.rs" + +[dependencies] +jsonschema = { version = "0.49", default-features = false } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" + +[lints.rust] +unsafe_code = "forbid" + +[lints.clippy] +all = "warn" +pedantic = "warn" diff --git a/conformance/runner/README.md b/conformance/runner/README.md new file mode 100644 index 0000000..2a39a0c --- /dev/null +++ b/conformance/runner/README.md @@ -0,0 +1,31 @@ +# PromptSyntax conformance runner + +This stable-Rust binary checks the language-independent corpus owned by this repository. +It does not depend on either PromptSyntax parser implementation and never embeds expected +answers. The specification and versioned vectors remain the oracle. + +From the repository root: + +```bash +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + check-requirements conformance/requirements.json + +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + check-schema schemas/prompt-trace-0.3-draft.schema.json + +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + validate-instance schemas/prompt-trace-0.3-draft.schema.json \ + examples/prompt-trace-0.3-draft.example.json + +cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . +``` + +The command emits deterministic JSON and exits with: + +- `0` when the input passes the implemented checks; +- `1` when the input produces conformance diagnostics; or +- `2` for invalid command usage or an internal report-serialization error. + +The runner performs no network requests and needs no provider SDK, model credentials, +Python, Node, or Bun. It is bootstrap tooling for a working draft, not a certification +utility. diff --git a/conformance/runner/src/lib.rs b/conformance/runner/src/lib.rs new file mode 100644 index 0000000..57a33fa --- /dev/null +++ b/conformance/runner/src/lib.rs @@ -0,0 +1,694 @@ +#![forbid(unsafe_code)] + +use std::collections::{BTreeMap, HashSet}; +use std::path::{Component, Path}; + +use jsonschema::PatternOptions; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +mod suite; +mod trace; + +pub use suite::{SuiteReport, run_suite}; + +const SUPPORTED_FORMAT: &str = "0.1-draft"; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RequirementsDocument { + pub format_version: String, + pub spec: SpecBaseline, + pub requirements: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SpecBaseline { + pub path: String, + pub commit: String, + pub declared_version: String, + pub baseline_status: BaselineStatus, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum BaselineStatus { + WorkingDraft, + Frozen, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Requirement { + pub id: String, + pub layer: Layer, + pub normative_status: NormativeStatus, + pub strength: Strength, + pub source: RequirementSource, + pub statement: String, + pub testability: Testability, + pub targets: Vec, + #[serde(default)] + pub blocked_by: Vec, + #[serde(default)] + pub notes: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RequirementSource { + pub path: String, + pub section: String, + pub candidate_anchor: String, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Layer { + Meta, + Core, + Capabilities, + Execution, + Trace, +} + +impl Layer { + const fn id_component(self) -> &'static str { + match self { + Self::Meta => "META", + Self::Core => "CORE", + Self::Capabilities => "CAP", + Self::Execution => "EXEC", + Self::Trace => "TRACE", + } + } + + const fn report_name(self) -> &'static str { + match self { + Self::Meta => "meta", + Self::Core => "core", + Self::Capabilities => "capabilities", + Self::Execution => "execution", + Self::Trace => "trace", + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum NormativeStatus { + Settled, + Proposed, + Open, +} + +impl NormativeStatus { + const fn report_name(self) -> &'static str { + match self { + Self::Settled => "settled", + Self::Proposed => "proposed", + Self::Open => "open", + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Strength { + Must, + MustNot, + Should, + ShouldNot, + May, + Invariant, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Testability { + Automatic, + Manual, + Profile, + Blocked, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum Target { + Corpus, + CoreParser, + CapabilityDocument, + ExecutionResolver, + TraceDocument, + TraceProducer, + InteractiveUi, + Documentation, + OversightProfile, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct Diagnostic { + pub code: String, + pub pointer: String, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct CheckReport { + pub format_version: String, + pub conformant: bool, + pub requirement_count: usize, + pub counts: BTreeMap, + pub diagnostics: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct ValidationReport { + pub kind: String, + pub conformant: bool, + pub diagnostics: Vec, +} + +impl ValidationReport { + #[must_use] + pub fn invalid_input(kind: &str, code: &str, message: impl Into) -> Self { + Self { + kind: kind.to_owned(), + conformant: false, + diagnostics: vec![Diagnostic { + code: code.to_owned(), + pointer: String::new(), + message: message.into(), + }], + } + } +} + +impl CheckReport { + #[must_use] + pub fn invalid_input(code: &str, message: impl Into) -> Self { + Self { + format_version: SUPPORTED_FORMAT.to_owned(), + conformant: false, + requirement_count: 0, + counts: BTreeMap::new(), + diagnostics: vec![Diagnostic { + code: code.to_owned(), + pointer: String::new(), + message: message.into(), + }], + } + } +} + +#[must_use] +pub fn check_requirements_json(input: &[u8]) -> CheckReport { + match serde_json::from_slice::(input) { + Ok(document) => check_requirements(&document), + Err(error) => CheckReport::invalid_input("CORPUS_JSON_INVALID", error.to_string()), + } +} + +#[must_use] +pub fn check_schema_json(input: &[u8]) -> ValidationReport { + let schema = match serde_json::from_slice::(input) { + Ok(schema) => schema, + Err(error) => { + return ValidationReport::invalid_input( + "schema", + "SCHEMA_JSON_INVALID", + error.to_string(), + ); + } + }; + + match jsonschema::meta::validate(&schema) { + Ok(()) => ValidationReport { + kind: "schema".to_owned(), + conformant: true, + diagnostics: Vec::new(), + }, + Err(error) => { + ValidationReport::invalid_input("schema", "SCHEMA_META_INVALID", error.to_string()) + } + } +} + +#[must_use] +pub fn validate_instance_json(schema_input: &[u8], instance_input: &[u8]) -> ValidationReport { + let schema = match serde_json::from_slice::(schema_input) { + Ok(schema) => schema, + Err(error) => { + return ValidationReport::invalid_input( + "instance", + "SCHEMA_JSON_INVALID", + error.to_string(), + ); + } + }; + let instance = match serde_json::from_slice::(instance_input) { + Ok(instance) => instance, + Err(error) => { + return ValidationReport::invalid_input( + "instance", + "INSTANCE_JSON_INVALID", + error.to_string(), + ); + } + }; + + if let Err(error) = jsonschema::meta::validate(&schema) { + return ValidationReport::invalid_input( + "instance", + "SCHEMA_META_INVALID", + error.to_string(), + ); + } + + let validator = match jsonschema::draft202012::options() + .with_pattern_options(PatternOptions::regex()) + .should_validate_formats(true) + .build(&schema) + { + Ok(validator) => validator, + Err(error) => { + return ValidationReport::invalid_input( + "instance", + "SCHEMA_BUILD_FAILED", + error.to_string(), + ); + } + }; + + let mut diagnostics = validator + .iter_errors(&instance) + .map(|error| Diagnostic { + code: "SCHEMA_INSTANCE_INVALID".to_owned(), + pointer: error.instance_path().to_string(), + message: error.to_string(), + }) + .collect::>(); + diagnostics.sort_by(|left, right| { + left.pointer + .cmp(&right.pointer) + .then_with(|| left.message.cmp(&right.message)) + }); + + ValidationReport { + kind: "instance".to_owned(), + conformant: diagnostics.is_empty(), + diagnostics, + } +} + +#[must_use] +pub fn check_requirements(document: &RequirementsDocument) -> CheckReport { + let mut diagnostics = Vec::new(); + + if document.format_version != SUPPORTED_FORMAT { + push_diagnostic( + &mut diagnostics, + "CORPUS_FORMAT_UNSUPPORTED", + "/format_version", + format!( + "expected format {SUPPORTED_FORMAT}, got {}", + document.format_version + ), + ); + } + + if !is_lower_hex_commit(&document.spec.commit) { + push_diagnostic( + &mut diagnostics, + "CORPUS_SPEC_COMMIT_INVALID", + "/spec/commit", + "commit must contain exactly 40 lowercase hexadecimal characters", + ); + } + + if !is_safe_relative_path(&document.spec.path) { + push_diagnostic( + &mut diagnostics, + "CORPUS_SOURCE_PATH_INVALID", + "/spec/path", + "spec path must be a non-empty relative path without parent traversal", + ); + } + + if document.requirements.is_empty() { + push_diagnostic( + &mut diagnostics, + "CORPUS_REQUIREMENTS_EMPTY", + "/requirements", + "at least one requirement is required", + ); + } + + let mut ids = HashSet::new(); + for (index, requirement) in document.requirements.iter().enumerate() { + check_requirement(requirement, index, &mut ids, &mut diagnostics); + } + + let mut counts = BTreeMap::new(); + for requirement in &document.requirements { + increment_count( + &mut counts, + format!("layer:{}", requirement.layer.report_name()), + ); + increment_count( + &mut counts, + format!("status:{}", requirement.normative_status.report_name()), + ); + } + + diagnostics.sort_by(|left, right| { + left.pointer + .cmp(&right.pointer) + .then_with(|| left.code.cmp(&right.code)) + }); + + CheckReport { + format_version: document.format_version.clone(), + conformant: diagnostics.is_empty(), + requirement_count: document.requirements.len(), + counts, + diagnostics, + } +} + +fn check_requirement( + requirement: &Requirement, + index: usize, + ids: &mut HashSet, + diagnostics: &mut Vec, +) { + let base = format!("/requirements/{index}"); + let expected_prefix = format!("PS-{}-", requirement.layer.id_component()); + + if !is_requirement_id(&requirement.id) || !requirement.id.starts_with(&expected_prefix) { + push_diagnostic( + diagnostics, + "CORPUS_REQUIREMENT_ID_INVALID", + format!("{base}/id"), + format!("requirement id must use uppercase ASCII and start with {expected_prefix}"), + ); + } + + if !ids.insert(requirement.id.clone()) { + push_diagnostic( + diagnostics, + "CORPUS_REQUIREMENT_ID_DUPLICATE", + format!("{base}/id"), + format!("duplicate requirement id {}", requirement.id), + ); + } + + if requirement.statement.trim().is_empty() { + push_diagnostic( + diagnostics, + "CORPUS_REQUIREMENT_STATEMENT_EMPTY", + format!("{base}/statement"), + "requirement statement must not be empty", + ); + } + + check_source(&requirement.source, &base, diagnostics); + check_targets(&requirement.targets, &base, diagnostics); + check_blockers(requirement, &base, diagnostics); +} + +fn check_source(source: &RequirementSource, base: &str, diagnostics: &mut Vec) { + if !is_safe_relative_path(&source.path) { + push_diagnostic( + diagnostics, + "CORPUS_SOURCE_PATH_INVALID", + format!("{base}/source/path"), + "source path must be relative and must not contain parent traversal", + ); + } + + if source.section.trim().is_empty() { + push_diagnostic( + diagnostics, + "CORPUS_SOURCE_SECTION_EMPTY", + format!("{base}/source/section"), + "source section must not be empty", + ); + } + + if !is_candidate_anchor(&source.candidate_anchor) { + push_diagnostic( + diagnostics, + "CORPUS_SOURCE_ANCHOR_INVALID", + format!("{base}/source/candidate_anchor"), + "candidate anchor must use lowercase ASCII letters, digits, and hyphens", + ); + } +} + +fn check_targets(targets: &[Target], base: &str, diagnostics: &mut Vec) { + if targets.is_empty() { + push_diagnostic( + diagnostics, + "CORPUS_TARGETS_EMPTY", + format!("{base}/targets"), + "at least one target is required", + ); + } + + let unique_targets = targets.iter().copied().collect::>(); + if unique_targets.len() != targets.len() { + push_diagnostic( + diagnostics, + "CORPUS_TARGET_DUPLICATE", + format!("{base}/targets"), + "targets must be unique", + ); + } +} + +fn check_blockers(requirement: &Requirement, base: &str, diagnostics: &mut Vec) { + for (blocker_index, blocker) in requirement.blocked_by.iter().enumerate() { + if !is_blocker_id(blocker) { + push_diagnostic( + diagnostics, + "CORPUS_BLOCKER_ID_INVALID", + format!("{base}/blocked_by/{blocker_index}"), + "blocker id must use the form A01", + ); + } + } + + let unique_blockers = requirement.blocked_by.iter().collect::>(); + if unique_blockers.len() != requirement.blocked_by.len() { + push_diagnostic( + diagnostics, + "CORPUS_BLOCKER_DUPLICATE", + format!("{base}/blocked_by"), + "blocker ids must be unique", + ); + } + + if requirement.testability == Testability::Blocked && requirement.blocked_by.is_empty() { + push_diagnostic( + diagnostics, + "CORPUS_BLOCKED_WITHOUT_REASON", + format!("{base}/blocked_by"), + "a blocked requirement must identify at least one audit blocker", + ); + } + + if requirement.testability != Testability::Blocked && !requirement.blocked_by.is_empty() { + push_diagnostic( + diagnostics, + "CORPUS_BLOCKER_ON_UNBLOCKED_REQUIREMENT", + format!("{base}/blocked_by"), + "only blocked requirements may declare audit blockers", + ); + } +} + +fn increment_count(counts: &mut BTreeMap, key: String) { + *counts.entry(key).or_default() += 1; +} + +fn push_diagnostic( + diagnostics: &mut Vec, + code: &str, + pointer: impl Into, + message: impl Into, +) { + diagnostics.push(Diagnostic { + code: code.to_owned(), + pointer: pointer.into(), + message: message.into(), + }); +} + +fn is_lower_hex_commit(value: &str) -> bool { + value.len() == 40 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn is_requirement_id(value: &str) -> bool { + value.starts_with("PS-") + && value.len() > 3 + && value + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'-') + && !value.ends_with('-') + && !value.contains("--") +} + +fn is_candidate_anchor(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && !value.starts_with('-') + && !value.ends_with('-') + && !value.contains("--") +} + +fn is_blocker_id(value: &str) -> bool { + let bytes = value.as_bytes(); + bytes.len() == 3 && bytes[0] == b'A' && bytes[1].is_ascii_digit() && bytes[2].is_ascii_digit() +} + +fn is_safe_relative_path(value: &str) -> bool { + let path = Path::new(value); + !value.is_empty() + && !path.is_absolute() + && path + .components() + .all(|component| matches!(component, Component::Normal(_) | Component::CurDir)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn valid_requirement(id: &str) -> Requirement { + Requirement { + id: id.to_owned(), + layer: Layer::Core, + normative_status: NormativeStatus::Settled, + strength: Strength::Must, + source: RequirementSource { + path: "spec/SPEC.md".to_owned(), + section: "13".to_owned(), + candidate_anchor: "ps-core-test-001".to_owned(), + }, + statement: "The behavior is explicit.".to_owned(), + testability: Testability::Automatic, + targets: vec![Target::CoreParser], + blocked_by: Vec::new(), + notes: None, + } + } + + fn valid_document() -> RequirementsDocument { + RequirementsDocument { + format_version: SUPPORTED_FORMAT.to_owned(), + spec: SpecBaseline { + path: "spec/SPEC.md".to_owned(), + commit: "a".repeat(40), + declared_version: "0.3-draft".to_owned(), + baseline_status: BaselineStatus::WorkingDraft, + }, + requirements: vec![valid_requirement("PS-CORE-TEST-001")], + } + } + + #[test] + fn accepts_valid_document() { + let report = check_requirements(&valid_document()); + assert!(report.conformant, "{:?}", report.diagnostics); + assert_eq!(report.requirement_count, 1); + assert_eq!(report.counts["layer:core"], 1); + assert_eq!(report.counts["status:settled"], 1); + } + + #[test] + fn rejects_duplicate_ids() { + let mut document = valid_document(); + document + .requirements + .push(valid_requirement("PS-CORE-TEST-001")); + let report = check_requirements(&document); + assert!(!report.conformant); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORPUS_REQUIREMENT_ID_DUPLICATE") + ); + } + + #[test] + fn rejects_id_that_disagrees_with_layer() { + let mut document = valid_document(); + document.requirements[0].id = "PS-TRACE-TEST-001".to_owned(); + let report = check_requirements(&document); + assert!(!report.conformant); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORPUS_REQUIREMENT_ID_INVALID") + ); + } + + #[test] + fn blocked_requirement_names_audit_finding() { + let mut document = valid_document(); + document.requirements[0].testability = Testability::Blocked; + let report = check_requirements(&document); + assert!(!report.conformant); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORPUS_BLOCKED_WITHOUT_REASON") + ); + } + + #[test] + fn rejects_unknown_json_fields() { + let json = br#"{ + "format_version": "0.1-draft", + "spec": { + "path": "spec/SPEC.md", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "declared_version": "draft", + "baseline_status": "working-draft", + "unexpected": true + }, + "requirements": [] + }"#; + let report = check_requirements_json(json); + assert!(!report.conformant); + assert_eq!(report.diagnostics[0].code, "CORPUS_JSON_INVALID"); + } + + #[test] + fn validates_schema_and_instance() { + let schema = br#"{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": ["value"], + "properties": { "value": { "type": "integer", "minimum": 0 } } + }"#; + assert!(check_schema_json(schema).conformant); + assert!(validate_instance_json(schema, br#"{"value": 1}"#).conformant); + + let invalid = validate_instance_json(schema, br#"{"value": -1}"#); + assert!(!invalid.conformant); + assert_eq!(invalid.diagnostics[0].code, "SCHEMA_INSTANCE_INVALID"); + assert_eq!(invalid.diagnostics[0].pointer, "/value"); + } +} diff --git a/conformance/runner/src/main.rs b/conformance/runner/src/main.rs new file mode 100644 index 0000000..dce277c --- /dev/null +++ b/conformance/runner/src/main.rs @@ -0,0 +1,120 @@ +#![forbid(unsafe_code)] + +use std::env; +use std::fs; +use std::process::ExitCode; + +use promptsyntax_conformance::{ + CheckReport, ValidationReport, check_requirements_json, check_schema_json, run_suite, + validate_instance_json, +}; +use serde::Serialize; + +fn main() -> ExitCode { + let mut args = env::args_os(); + let _program = args.next(); + let Some(command) = args.next() else { + return usage(); + }; + match command.to_str() { + Some("check-requirements") => { + let Some(path) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let report = match fs::read(&path) { + Ok(input) => check_requirements_json(&input), + Err(error) => CheckReport::invalid_input("CORPUS_READ_FAILED", error.to_string()), + }; + emit_report(&report, report.conformant) + } + Some("check-schema") => { + let Some(path) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let report = match fs::read(&path) { + Ok(input) => check_schema_json(&input), + Err(error) => ValidationReport::invalid_input( + "schema", + "SCHEMA_READ_FAILED", + error.to_string(), + ), + }; + emit_report(&report, report.conformant) + } + Some("validate-instance") => { + let Some(schema_path) = args.next() else { + return usage(); + }; + let Some(instance_path) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let schema = match fs::read(&schema_path) { + Ok(input) => input, + Err(error) => { + let report = ValidationReport::invalid_input( + "instance", + "SCHEMA_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let instance = match fs::read(&instance_path) { + Ok(input) => input, + Err(error) => { + let report = ValidationReport::invalid_input( + "instance", + "INSTANCE_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let report = validate_instance_json(&schema, &instance); + emit_report(&report, report.conformant) + } + Some("run-suite") => { + let Some(root) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let report = run_suite(std::path::Path::new(&root)); + emit_report(&report, report.conformant) + } + _ => usage(), + } +} + +fn emit_report(report: &impl Serialize, conformant: bool) -> ExitCode { + match serde_json::to_string_pretty(report) { + Ok(json) => println!("{json}"), + Err(error) => { + eprintln!("failed to serialize conformance report: {error}"); + return ExitCode::from(2); + } + } + + if conformant { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } +} + +fn usage() -> ExitCode { + eprintln!( + "usage:\n ps-conformance check-requirements \n ps-conformance check-schema \n ps-conformance validate-instance \n ps-conformance run-suite " + ); + ExitCode::from(2) +} diff --git a/conformance/runner/src/suite.rs b/conformance/runner/src/suite.rs new file mode 100644 index 0000000..fe6a8ff --- /dev/null +++ b/conformance/runner/src/suite.rs @@ -0,0 +1,967 @@ +use std::collections::{BTreeSet, HashSet}; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::trace::{ProducerTranscript, validate_trace}; +use crate::{Diagnostic, check_requirements_json, check_schema_json, validate_instance_json}; + +const FORMAT_VERSION: &str = "0.1-draft"; + +#[derive(Debug, Serialize)] +pub struct SuiteReport { + pub format_version: String, + pub suite: VersionedIdentity, + pub spec: ReportSpec, + pub profile: VersionedIdentity, + pub conformant: bool, + pub summary: SuiteSummary, + pub coverage: CoverageReport, + pub families: Vec, + pub diagnostics: Vec, +} + +#[derive(Debug, Serialize)] +pub struct VersionedIdentity { + pub id: String, + pub version: String, + pub status: String, +} + +#[derive(Debug, Serialize)] +pub struct ReportSpec { + pub commit: String, + pub declared_version: String, +} + +#[derive(Debug, Default, Serialize)] +pub struct SuiteSummary { + pub families: usize, + pub cases: usize, + pub passed: usize, + pub failed: usize, + pub requirements_required: usize, + pub requirements_covered: usize, +} + +#[derive(Debug, Default, Serialize)] +pub struct CoverageReport { + pub required: Vec, + pub covered: Vec, + pub missing: Vec, +} + +#[derive(Debug, Serialize)] +pub struct FamilyReport { + pub id: String, + pub target: String, + pub status: String, + pub cases: Vec, +} + +#[derive(Debug, Serialize)] +pub struct CaseReport { + pub id: String, + pub passed: bool, + pub expected_conformant: bool, + pub actual_conformant: bool, + pub expected_diagnostics: Vec, + pub actual_diagnostics: Vec, + pub requirements: Vec, +} + +#[derive(Debug, Deserialize)] +struct Manifest { + format_version: String, + suite: SuiteMetadata, + spec: ManifestSpec, + files: ManifestFiles, + families: Vec, + claims: Vec, +} + +#[derive(Debug, Deserialize)] +struct SuiteMetadata { + id: String, + version: String, + status: String, +} + +#[derive(Debug, Deserialize)] +struct ManifestSpec { + path: String, + commit: String, + declared_version: String, + baseline_status: String, +} + +#[derive(Debug, Deserialize)] +struct ManifestFiles { + requirements: String, + requirements_schema: String, + diagnostics: String, + diagnostics_schema: String, + trace_schema: String, + family_schema: String, + transcript_schema: String, + profile: String, + profile_schema: String, + report_schema: String, + core_cases: String, + core_cases_schema: String, + adapter_result_schema: String, + implementations_lock: String, + implementations_schema: String, + candidate_report: String, +} + +#[derive(Debug, Deserialize)] +struct ManifestFamily { + id: String, + layer: String, + target: String, + version: String, + path: String, + status: String, +} + +#[derive(Debug, Deserialize)] +struct ManifestClaim { + profile: String, + families: Vec, + status: String, +} + +#[derive(Debug, Deserialize)] +struct Profile { + id: String, + version: String, + status: String, + requirements: Vec, +} + +#[derive(Debug, Deserialize)] +struct CaseFamily { + format_version: String, + id: String, + version: String, + status: String, + profile: String, + target: String, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct Case { + id: String, + requirements: Vec, + trace: String, + #[serde(default)] + patch: Vec, + transcript: Option, + #[serde(default)] + transcript_patch: Vec, + expected: Expected, +} + +#[derive(Debug, Deserialize)] +struct Expected { + conformant: bool, + diagnostics: Vec, +} + +#[derive(Debug, Deserialize)] +struct PatchOperation { + op: String, + path: String, + value: Option, +} + +#[must_use] +pub fn run_suite(root: &Path) -> SuiteReport { + match run_suite_inner(root) { + Ok(report) => report, + Err(diagnostic) => failure_report(diagnostic), + } +} + +#[allow(clippy::too_many_lines)] +fn run_suite_inner(root: &Path) -> Result { + let manifest_value = read_json(root, "conformance/manifest.json")?; + let manifest_schema = read_json(root, "conformance/manifest.schema.json")?; + ensure_schema(&manifest_schema, "/conformance/manifest.schema.json")?; + ensure_instance( + &manifest_schema, + &manifest_value, + "/conformance/manifest.json", + )?; + let manifest: Manifest = deserialize_value(manifest_value, "/conformance/manifest.json")?; + + let mut suite_diagnostics = Vec::new(); + if manifest.format_version != FORMAT_VERSION { + push( + &mut suite_diagnostics, + "SUITE_FORMAT_UNSUPPORTED", + "/format_version", + format!("suite format must be {FORMAT_VERSION}"), + ); + } + if manifest.spec.baseline_status == "frozen" && manifest.suite.status != "released" { + push( + &mut suite_diagnostics, + "SUITE_BASELINE_STATUS_INVALID", + "/spec/baseline_status", + "a frozen baseline requires a released suite status", + ); + } + if !is_safe_relative_path(&manifest.spec.path) { + push( + &mut suite_diagnostics, + "SUITE_PATH_INVALID", + "/spec/path", + "specification path is not a safe repository-relative path", + ); + } + + let requirements_schema = checked_schema(root, &manifest.files.requirements_schema)?; + let requirements_value = read_json(root, &manifest.files.requirements)?; + append_validation( + &mut suite_diagnostics, + validate_value(&requirements_schema, &requirements_value), + &manifest.files.requirements, + ); + let requirements_bytes = serde_json::to_vec(&requirements_value).map_err(|error| { + diagnostic( + "SUITE_JSON_INVALID", + &manifest.files.requirements, + error.to_string(), + ) + })?; + suite_diagnostics.extend(check_requirements_json(&requirements_bytes).diagnostics); + let requirement_ids = requirements_value + .get("requirements") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|item| item.get("id").and_then(Value::as_str)) + .map(str::to_owned) + .collect::>(); + + let diagnostics_schema = checked_schema(root, &manifest.files.diagnostics_schema)?; + let diagnostics_value = read_json(root, &manifest.files.diagnostics)?; + append_validation( + &mut suite_diagnostics, + validate_value(&diagnostics_schema, &diagnostics_value), + &manifest.files.diagnostics, + ); + let diagnostic_codes = registry_codes(&diagnostics_value, &mut suite_diagnostics); + + let trace_schema = checked_schema(root, &manifest.files.trace_schema)?; + let family_schema = checked_schema(root, &manifest.files.family_schema)?; + let transcript_schema = checked_schema(root, &manifest.files.transcript_schema)?; + let profile_schema = checked_schema(root, &manifest.files.profile_schema)?; + let report_schema = checked_schema(root, &manifest.files.report_schema)?; + let core_cases_schema = checked_schema(root, &manifest.files.core_cases_schema)?; + let _adapter_result_schema = checked_schema(root, &manifest.files.adapter_result_schema)?; + + let core_cases = read_json(root, &manifest.files.core_cases)?; + append_validation( + &mut suite_diagnostics, + validate_value(&core_cases_schema, &core_cases), + &manifest.files.core_cases, + ); + let implementations_schema = checked_schema(root, &manifest.files.implementations_schema)?; + let implementations_lock = read_json(root, &manifest.files.implementations_lock)?; + append_validation( + &mut suite_diagnostics, + validate_value(&implementations_schema, &implementations_lock), + &manifest.files.implementations_lock, + ); + let candidate_report = read_json(root, &manifest.files.candidate_report)?; + append_validation( + &mut suite_diagnostics, + validate_value(&report_schema, &candidate_report), + &manifest.files.candidate_report, + ); + + let profile_value = read_json(root, &manifest.files.profile)?; + append_validation( + &mut suite_diagnostics, + validate_value(&profile_schema, &profile_value), + &manifest.files.profile, + ); + let profile: Profile = deserialize_value(profile_value, &manifest.files.profile)?; + let profile_requirements = profile + .requirements + .iter() + .cloned() + .collect::>(); + for requirement in &profile_requirements { + if !requirement_ids.contains(requirement) { + push( + &mut suite_diagnostics, + "SUITE_PROFILE_REQUIREMENT_UNKNOWN", + &manifest.files.profile, + format!("profile requirement {requirement} is absent from the inventory"), + ); + } + } + + validate_claims(&manifest, &profile, &mut suite_diagnostics); + + let mut case_ids = HashSet::new(); + let mut covered_requirements = BTreeSet::new(); + let mut family_reports = Vec::new(); + + for family_entry in &manifest.families { + let family_value = read_json(root, &family_entry.path)?; + let family_validation = validate_value(&family_schema, &family_value); + append_validation( + &mut suite_diagnostics, + family_validation.clone(), + &family_entry.path, + ); + if !family_validation.conformant { + continue; + } + let family: CaseFamily = deserialize_value(family_value, &family_entry.path)?; + validate_family_metadata(family_entry, &family, &profile, &mut suite_diagnostics); + + let mut case_reports = Vec::new(); + for case in &family.cases { + if !case_ids.insert(case.id.clone()) { + push( + &mut suite_diagnostics, + "SUITE_CASE_ID_DUPLICATE", + &family_entry.path, + format!("case id {} occurs more than once", case.id), + ); + } + validate_case_requirements( + case, + &requirement_ids, + &profile_requirements, + &family_entry.path, + &mut suite_diagnostics, + ); + let report = run_case( + root, + &family, + case, + &trace_schema, + &transcript_schema, + &diagnostic_codes, + &family_entry.path, + &mut suite_diagnostics, + ); + if report.passed { + covered_requirements.extend(report.requirements.iter().cloned()); + } + case_reports.push(report); + } + case_reports.sort_by(|left, right| left.id.cmp(&right.id)); + family_reports.push(FamilyReport { + id: family.id, + target: family.target, + status: family.status, + cases: case_reports, + }); + } + family_reports.sort_by(|left, right| left.id.cmp(&right.id)); + + let missing_requirements = profile_requirements + .difference(&covered_requirements) + .cloned() + .collect::>(); + for requirement in &missing_requirements { + push( + &mut suite_diagnostics, + "SUITE_PROFILE_COVERAGE_MISSING", + "/coverage/missing", + format!("profile requirement {requirement} has no passing case"), + ); + } + + suite_diagnostics.sort_by(|left, right| { + left.pointer + .cmp(&right.pointer) + .then_with(|| left.code.cmp(&right.code)) + .then_with(|| left.message.cmp(&right.message)) + }); + let cases = family_reports + .iter() + .map(|family| family.cases.len()) + .sum::(); + let passed = family_reports + .iter() + .flat_map(|family| &family.cases) + .filter(|case| case.passed) + .count(); + let failed = cases.saturating_sub(passed); + let conformant = failed == 0 && missing_requirements.is_empty() && suite_diagnostics.is_empty(); + let required = profile_requirements.iter().cloned().collect::>(); + let covered = covered_requirements + .intersection(&profile_requirements) + .cloned() + .collect::>(); + + let mut report = SuiteReport { + format_version: FORMAT_VERSION.to_owned(), + suite: VersionedIdentity { + id: manifest.suite.id, + version: manifest.suite.version, + status: manifest.suite.status, + }, + spec: ReportSpec { + commit: manifest.spec.commit, + declared_version: manifest.spec.declared_version, + }, + profile: VersionedIdentity { + id: profile.id, + version: profile.version, + status: profile.status, + }, + conformant, + summary: SuiteSummary { + families: family_reports.len(), + cases, + passed, + failed, + requirements_required: required.len(), + requirements_covered: covered.len(), + }, + coverage: CoverageReport { + required, + covered, + missing: missing_requirements, + }, + families: family_reports, + diagnostics: suite_diagnostics, + }; + + let report_value = serde_json::to_value(&report).map_err(|error| { + diagnostic( + "SUITE_REPORT_INVALID", + &manifest.files.report_schema, + error.to_string(), + ) + })?; + let report_validation = validate_value(&report_schema, &report_value); + if !report_validation.conformant { + report.conformant = false; + report.diagnostics.push(diagnostic( + "SUITE_REPORT_INVALID", + &manifest.files.report_schema, + "generated report does not satisfy its schema", + )); + } + Ok(report) +} + +#[allow(clippy::too_many_arguments)] +fn run_case( + root: &Path, + family: &CaseFamily, + case: &Case, + trace_schema: &Value, + transcript_schema: &Value, + diagnostic_codes: &HashSet, + family_path: &str, + suite_diagnostics: &mut Vec, +) -> CaseReport { + let mut actual_diagnostics = Vec::new(); + let trace_result = read_json(root, &case.trace).and_then(|mut value| { + apply_patch(&mut value, &case.patch).map_err(|message| { + diagnostic( + "CASE_PATCH_INVALID", + family_path, + format!("case {} trace patch failed: {message}", case.id), + ) + })?; + Ok(value) + }); + + if let Ok(trace) = trace_result { + let structural = validate_value(trace_schema, &trace); + actual_diagnostics.extend(structural.diagnostics); + if structural.conformant { + let transcript = load_transcript( + root, + family, + case, + transcript_schema, + &mut actual_diagnostics, + ); + if family.target != "trace-producer" || transcript.is_some() { + actual_diagnostics.extend(validate_trace(&trace, transcript.as_ref())); + } + } + } else if let Err(error) = trace_result { + actual_diagnostics.push(error); + } + + let actual_codes = actual_diagnostics + .iter() + .map(|item| item.code.clone()) + .collect::>(); + let mut expected_codes = case.expected.diagnostics.clone(); + expected_codes.sort(); + expected_codes.dedup(); + let actual_codes = actual_codes.into_iter().collect::>(); + let actual_conformant = actual_codes.is_empty(); + let passed = actual_conformant == case.expected.conformant && actual_codes == expected_codes; + + for code in expected_codes.iter().chain(&actual_codes) { + if !diagnostic_codes.contains(code) { + push( + suite_diagnostics, + "SUITE_DIAGNOSTIC_UNREGISTERED", + family_path, + format!("case {} uses unregistered diagnostic {code}", case.id), + ); + } + } + + let mut requirements = case.requirements.clone(); + requirements.sort(); + requirements.dedup(); + CaseReport { + id: case.id.clone(), + passed, + expected_conformant: case.expected.conformant, + actual_conformant, + expected_diagnostics: expected_codes, + actual_diagnostics: actual_codes, + requirements, + } +} + +fn load_transcript( + root: &Path, + family: &CaseFamily, + case: &Case, + transcript_schema: &Value, + diagnostics: &mut Vec, +) -> Option { + if family.target != "trace-producer" { + return None; + } + let Some(path) = &case.transcript else { + diagnostics.push(diagnostic( + "SUITE_PRODUCER_TRANSCRIPT_REQUIRED", + &case.id, + "trace-producer case has no deterministic transcript", + )); + return None; + }; + let mut value = match read_json(root, path) { + Ok(value) => value, + Err(error) => { + diagnostics.push(error); + return None; + } + }; + if let Err(message) = apply_patch(&mut value, &case.transcript_patch) { + diagnostics.push(diagnostic( + "CASE_PATCH_INVALID", + path, + format!("case {} transcript patch failed: {message}", case.id), + )); + return None; + } + let validation = validate_value(transcript_schema, &value); + diagnostics.extend(validation.diagnostics); + if !validation.conformant { + return None; + } + match serde_json::from_value(value) { + Ok(transcript) => Some(transcript), + Err(error) => { + diagnostics.push(diagnostic("INSTANCE_JSON_INVALID", path, error.to_string())); + None + } + } +} + +fn validate_family_metadata( + entry: &ManifestFamily, + family: &CaseFamily, + profile: &Profile, + diagnostics: &mut Vec, +) { + let matches = family.format_version == FORMAT_VERSION + && entry.id == family.id + && entry.layer == "trace" + && entry.target == family.target + && entry.version == family.version + && entry.status == family.status + && family.profile == profile.id; + if !matches { + push( + diagnostics, + "SUITE_FAMILY_METADATA_MISMATCH", + &entry.path, + "manifest family metadata disagrees with the family document or active profile", + ); + } +} + +fn validate_case_requirements( + case: &Case, + requirement_ids: &HashSet, + profile_requirements: &BTreeSet, + family_path: &str, + diagnostics: &mut Vec, +) { + for requirement in &case.requirements { + if !requirement_ids.contains(requirement) { + push( + diagnostics, + "SUITE_CASE_REQUIREMENT_UNKNOWN", + family_path, + format!("case {} cites unknown requirement {requirement}", case.id), + ); + } else if !profile_requirements.contains(requirement) { + push( + diagnostics, + "SUITE_CASE_REQUIREMENT_OUTSIDE_PROFILE", + family_path, + format!( + "case {} cites requirement {requirement} outside the active profile", + case.id + ), + ); + } + } +} + +fn validate_claims(manifest: &Manifest, profile: &Profile, diagnostics: &mut Vec) { + let family_ids = manifest + .families + .iter() + .map(|family| family.id.as_str()) + .collect::>(); + for (claim_index, claim) in manifest.claims.iter().enumerate() { + if claim.profile != profile.id || claim.status != profile.status { + push( + diagnostics, + "SUITE_CLAIM_PROFILE_MISMATCH", + format!("/claims/{claim_index}"), + "claim profile or status disagrees with the profile document", + ); + } + for family in &claim.families { + if !family_ids.contains(family.as_str()) { + push( + diagnostics, + "SUITE_CLAIM_FAMILY_UNKNOWN", + format!("/claims/{claim_index}/families"), + format!("claim cites unknown family {family}"), + ); + } + } + } +} + +fn registry_codes(value: &Value, diagnostics: &mut Vec) -> HashSet { + let mut codes = HashSet::new(); + for (index, item) in value + .get("diagnostics") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + if let Some(code) = item.get("code").and_then(Value::as_str) { + if !codes.insert(code.to_owned()) { + push( + diagnostics, + "SUITE_DIAGNOSTIC_DUPLICATE", + format!("/diagnostics/{index}/code"), + format!("diagnostic code {code} occurs more than once"), + ); + } + } + } + codes +} + +fn checked_schema(root: &Path, relative: &str) -> Result { + let schema = read_json(root, relative)?; + ensure_schema(&schema, relative)?; + Ok(schema) +} + +fn ensure_schema(schema: &Value, pointer: &str) -> Result<(), Diagnostic> { + let bytes = serde_json::to_vec(schema) + .map_err(|error| diagnostic("SCHEMA_JSON_INVALID", pointer, error.to_string()))?; + let report = check_schema_json(&bytes); + if report.conformant { + Ok(()) + } else { + Err(diagnostic( + "SCHEMA_META_INVALID", + pointer, + report + .diagnostics + .first() + .map_or("schema meta-validation failed", |item| { + item.message.as_str() + }), + )) + } +} + +fn ensure_instance(schema: &Value, instance: &Value, pointer: &str) -> Result<(), Diagnostic> { + let report = validate_value(schema, instance); + if report.conformant { + Ok(()) + } else { + Err(diagnostic( + "SCHEMA_INSTANCE_INVALID", + pointer, + report + .diagnostics + .first() + .map_or("instance validation failed", |item| item.message.as_str()), + )) + } +} + +fn validate_value(schema: &Value, instance: &Value) -> crate::ValidationReport { + let schema_bytes = serde_json::to_vec(schema).unwrap_or_default(); + let instance_bytes = serde_json::to_vec(instance).unwrap_or_default(); + validate_instance_json(&schema_bytes, &instance_bytes) +} + +fn append_validation( + diagnostics: &mut Vec, + report: crate::ValidationReport, + artifact: &str, +) { + diagnostics.extend(report.diagnostics.into_iter().map(|mut item| { + item.pointer = format!("{artifact}{}", item.pointer); + item + })); +} + +fn deserialize_value Deserialize<'de>>( + value: Value, + pointer: &str, +) -> Result { + serde_json::from_value(value) + .map_err(|error| diagnostic("SUITE_JSON_INVALID", pointer, error.to_string())) +} + +fn read_json(root: &Path, relative: &str) -> Result { + let path = resolve_path(root, relative)?; + let bytes = fs::read(&path) + .map_err(|error| diagnostic("SUITE_FILE_READ_FAILED", relative, error.to_string()))?; + serde_json::from_slice(&bytes) + .map_err(|error| diagnostic("SUITE_JSON_INVALID", relative, error.to_string())) +} + +fn resolve_path(root: &Path, relative: &str) -> Result { + if !is_safe_relative_path(relative) { + return Err(diagnostic( + "SUITE_PATH_INVALID", + relative, + "path must be repository-relative without parent traversal", + )); + } + let canonical_root = root + .canonicalize() + .map_err(|error| diagnostic("SUITE_FILE_READ_FAILED", relative, error.to_string()))?; + let joined = canonical_root.join(relative); + let canonical_path = joined + .canonicalize() + .map_err(|error| diagnostic("SUITE_FILE_READ_FAILED", relative, error.to_string()))?; + if !canonical_path.starts_with(&canonical_root) { + return Err(diagnostic( + "SUITE_PATH_INVALID", + relative, + "path resolves outside the repository root", + )); + } + Ok(canonical_path) +} + +fn is_safe_relative_path(value: &str) -> bool { + let path = Path::new(value); + !value.is_empty() + && !path.is_absolute() + && path + .components() + .all(|component| matches!(component, Component::Normal(_) | Component::CurDir)) +} + +fn apply_patch(document: &mut Value, operations: &[PatchOperation]) -> Result<(), String> { + for operation in operations { + apply_operation(document, operation)?; + } + Ok(()) +} + +fn apply_operation(document: &mut Value, operation: &PatchOperation) -> Result<(), String> { + let (parent_pointer, token) = split_pointer(&operation.path)?; + let parent = document + .pointer_mut(&parent_pointer) + .ok_or_else(|| format!("patch parent {parent_pointer} does not exist"))?; + match (operation.op.as_str(), parent) { + ("add", Value::Object(object)) => { + let value = operation + .value + .clone() + .ok_or_else(|| "add operation requires value".to_owned())?; + object.insert(token, value); + Ok(()) + } + ("replace", Value::Object(object)) => { + if !object.contains_key(&token) { + return Err(format!("replace target {} does not exist", operation.path)); + } + let value = operation + .value + .clone() + .ok_or_else(|| "replace operation requires value".to_owned())?; + object.insert(token, value); + Ok(()) + } + ("remove", Value::Object(object)) => object + .remove(&token) + .map(|_| ()) + .ok_or_else(|| format!("remove target {} does not exist", operation.path)), + ("add", Value::Array(array)) => { + let value = operation + .value + .clone() + .ok_or_else(|| "add operation requires value".to_owned())?; + if token == "-" { + array.push(value); + return Ok(()); + } + let index = parse_index(&token, operation)?; + if index > array.len() { + return Err(format!("add index {index} is out of bounds")); + } + array.insert(index, value); + Ok(()) + } + ("replace", Value::Array(array)) => { + let index = parse_index(&token, operation)?; + let target = array + .get_mut(index) + .ok_or_else(|| format!("replace index {index} is out of bounds"))?; + *target = operation + .value + .clone() + .ok_or_else(|| "replace operation requires value".to_owned())?; + Ok(()) + } + ("remove", Value::Array(array)) => { + let index = parse_index(&token, operation)?; + if index >= array.len() { + return Err(format!("remove index {index} is out of bounds")); + } + array.remove(index); + Ok(()) + } + (op, _) => Err(format!("operation {op} cannot target {}", operation.path)), + } +} + +fn split_pointer(pointer: &str) -> Result<(String, String), String> { + let Some(index) = pointer.rfind('/') else { + return Err("patch path must be a JSON Pointer".to_owned()); + }; + let parent = pointer[..index].to_owned(); + let token = pointer[index + 1..].replace("~1", "/").replace("~0", "~"); + if token.is_empty() { + return Err("patch path token must not be empty".to_owned()); + } + Ok((parent, token)) +} + +fn parse_index(token: &str, operation: &PatchOperation) -> Result { + token + .parse() + .map_err(|_| format!("array token {token} in {} is not an index", operation.path)) +} + +fn failure_report(diagnostic: Diagnostic) -> SuiteReport { + SuiteReport { + format_version: FORMAT_VERSION.to_owned(), + suite: VersionedIdentity { + id: "promptsyntax-conformance".to_owned(), + version: "unknown".to_owned(), + status: "invalid".to_owned(), + }, + spec: ReportSpec { + commit: "0000000000000000000000000000000000000000".to_owned(), + declared_version: "unknown".to_owned(), + }, + profile: VersionedIdentity { + id: "unknown".to_owned(), + version: "unknown".to_owned(), + status: "invalid".to_owned(), + }, + conformant: false, + summary: SuiteSummary::default(), + coverage: CoverageReport::default(), + families: Vec::new(), + diagnostics: vec![diagnostic], + } +} + +fn diagnostic(code: &str, pointer: impl Into, message: impl Into) -> Diagnostic { + Diagnostic { + code: code.to_owned(), + pointer: pointer.into(), + message: message.into(), + } +} + +fn push( + diagnostics: &mut Vec, + code: &str, + pointer: impl Into, + message: impl Into, +) { + diagnostics.push(diagnostic(code, pointer, message)); +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn patch_supports_object_and_array_operations() { + let mut value = json!({"items": [1], "name": "before"}); + let operations = vec![ + PatchOperation { + op: "add".to_owned(), + path: "/items/-".to_owned(), + value: Some(json!(2)), + }, + PatchOperation { + op: "replace".to_owned(), + path: "/name".to_owned(), + value: Some(json!("after")), + }, + ]; + apply_patch(&mut value, &operations).expect("patch succeeds"); + assert_eq!(value, json!({"items": [1, 2], "name": "after"})); + } + + #[test] + fn repository_suite_matches_all_expected_results() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let report = run_suite(&root); + assert!(report.conformant, "{:#?}", report.diagnostics); + assert_eq!(report.summary.failed, 0); + assert!(report.summary.cases > 0); + } +} diff --git a/conformance/runner/src/trace.rs b/conformance/runner/src/trace.rs new file mode 100644 index 0000000..287c64b --- /dev/null +++ b/conformance/runner/src/trace.rs @@ -0,0 +1,942 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; + +use serde::Deserialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::Diagnostic; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProducerTranscript { + pub format_version: String, + pub turn: String, + pub inferences: Vec, + pub boundaries: Vec, + pub external_artifacts: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TranscriptInference { + pub id: String, + pub actual_request_utf8: String, + pub routing: TranscriptRouting, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TranscriptRouting { + pub outcome: String, + pub bound: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TranscriptBoundary { + pub id: String, + pub outcome: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ExternalArtifact { + pub sha256: String, + pub length: usize, + pub content_utf8: String, +} + +struct ArtifactIndex<'a> { + valid: HashMap<&'a str, &'a ExternalArtifact>, + invalid: HashSet<&'a str>, +} + +enum Reconstructed { + Bytes(Vec), + Unavailable, + Invalid, +} + +#[allow(clippy::too_many_lines)] +pub(crate) fn validate_trace( + trace: &Value, + transcript: Option<&ProducerTranscript>, +) -> Vec { + let mut diagnostics = Vec::new(); + let artifacts = build_artifact_index(transcript, &mut diagnostics); + + if let Some(transcript) = transcript { + if transcript.format_version != "0.1-draft" { + push( + &mut diagnostics, + "TRACE_PRODUCER_FORMAT_UNSUPPORTED", + "/transcript/format_version", + "producer transcript format must be 0.1-draft", + ); + } + if trace.get("turn").and_then(Value::as_str) != Some(transcript.turn.as_str()) { + push( + &mut diagnostics, + "TRACE_PRODUCER_TURN_MISMATCH", + "/turn", + "trace and producer transcript identify different turns", + ); + } + } + + let tier = trace + .get("tier") + .and_then(Value::as_str) + .unwrap_or_default(); + let content_mode = trace + .get("content_mode") + .and_then(Value::as_str) + .unwrap_or_default(); + let inline_threshold = trace + .pointer("/coverage/inline_threshold_bytes") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .unwrap_or(0); + + if content_mode == "materialized" && tier != "oversight" { + push( + &mut diagnostics, + "TRACE_R5_MODE_TIER_INVALID", + "/content_mode", + "materialized content mode is reserved for oversight exports", + ); + } + + let transcript_inferences = transcript + .map(|value| { + value + .inferences + .iter() + .map(|item| (item.id.as_str(), item)) + .collect::>() + }) + .unwrap_or_default(); + let transcript_boundaries = transcript + .map(|value| { + value + .boundaries + .iter() + .map(|item| (item.id.as_str(), item)) + .collect::>() + }) + .unwrap_or_default(); + + let mut seen_events = HashSet::new(); + let mut seen_transcript_inferences = HashSet::new(); + let mut seen_transcript_boundaries = HashSet::new(); + let events = trace + .get("events") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + + for (event_index, event) in events.iter().enumerate() { + let base = format!("/events/{event_index}"); + let event_id = event.get("id").and_then(Value::as_str).unwrap_or_default(); + if !seen_events.insert(event_id) { + push( + &mut diagnostics, + "TRACE_EVENT_ID_DUPLICATE", + format!("{base}/id"), + format!("event id {event_id} occurs more than once"), + ); + } + + match event.get("event").and_then(Value::as_str) { + Some("inference") => { + let transcript_fact = transcript_inferences.get(event_id).copied(); + if transcript.is_some() { + if transcript_fact.is_some() { + seen_transcript_inferences.insert(event_id); + } else { + push( + &mut diagnostics, + "TRACE_PRODUCER_INFERENCE_MISSING", + format!("{base}/id"), + "no independent producer fact exists for this inference", + ); + } + } + validate_inference( + event, + &base, + inline_threshold, + content_mode, + &artifacts, + transcript_fact, + &mut diagnostics, + ); + } + Some("boundary") => { + let transcript_fact = transcript_boundaries.get(event_id).copied(); + if transcript.is_some() { + if let Some(fact) = transcript_fact { + seen_transcript_boundaries.insert(event_id); + if event.get("outcome").and_then(Value::as_str) + != Some(fact.outcome.as_str()) + { + push( + &mut diagnostics, + "TRACE_PRODUCER_BOUNDARY_MISMATCH", + format!("{base}/outcome"), + "trace boundary outcome differs from the producer transcript", + ); + } + } else { + push( + &mut diagnostics, + "TRACE_PRODUCER_BOUNDARY_MISSING", + format!("{base}/id"), + "no independent producer fact exists for this boundary event", + ); + } + } + validate_boundary(event, &base, content_mode, &artifacts, &mut diagnostics); + } + _ => {} + } + } + + if let Some(transcript) = transcript { + for fact in &transcript.inferences { + if !seen_transcript_inferences.contains(fact.id.as_str()) { + push( + &mut diagnostics, + "TRACE_PRODUCER_TRACE_EVENT_MISSING", + "/events", + format!("producer inference {} is absent from the trace", fact.id), + ); + } + } + for fact in &transcript.boundaries { + if !seen_transcript_boundaries.contains(fact.id.as_str()) { + push( + &mut diagnostics, + "TRACE_PRODUCER_TRACE_EVENT_MISSING", + "/events", + format!("producer boundary {} is absent from the trace", fact.id), + ); + } + } + } + + diagnostics.sort_by(|left, right| { + left.pointer + .cmp(&right.pointer) + .then_with(|| left.code.cmp(&right.code)) + .then_with(|| left.message.cmp(&right.message)) + }); + diagnostics +} + +fn validate_inference( + inference: &Value, + base: &str, + inline_threshold: usize, + content_mode: &str, + artifacts: &ArtifactIndex<'_>, + transcript: Option<&TranscriptInference>, + diagnostics: &mut Vec, +) { + validate_provenance(inference, base, diagnostics); + validate_routing(inference, base, transcript, diagnostics); + + let Some(segments) = inference.get("segments").and_then(Value::as_array) else { + return; + }; + let reconstructed = reconstruct_segments( + segments, + base, + inline_threshold, + content_mode, + artifacts, + diagnostics, + ); + + if let Some(transcript) = transcript { + match reconstructed { + Reconstructed::Bytes(bytes) => { + if bytes != transcript.actual_request_utf8.as_bytes() { + push( + diagnostics, + "TRACE_R1_REQUEST_BYTES_MISMATCH", + format!("{base}/segments"), + "reconstructed segment bytes differ from the independently supplied request bytes", + ); + } + } + Reconstructed::Unavailable => push( + diagnostics, + "TRACE_R1_CONTENT_UNAVAILABLE", + format!("{base}/segments"), + "producer conformance cannot reconstruct all request bytes", + ), + Reconstructed::Invalid => {} + } + } +} + +fn validate_provenance(inference: &Value, base: &str, diagnostics: &mut Vec) { + let steps = inference + .get("steps") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let mut parents = HashMap::new(); + for (step_index, step) in steps.iter().enumerate() { + let Some(id) = step.get("id").and_then(Value::as_str) else { + continue; + }; + if parents + .insert(id, step.get("parent").and_then(Value::as_str)) + .is_some() + { + push( + diagnostics, + "TRACE_STEP_ID_DUPLICATE", + format!("{base}/steps/{step_index}/id"), + format!("step id {id} occurs more than once"), + ); + } + } + + for (step_index, step) in steps.iter().enumerate() { + if let Some(parent) = step.get("parent").and_then(Value::as_str) { + if !parents.contains_key(parent) { + push( + diagnostics, + "TRACE_R3_PARENT_NOT_FOUND", + format!("{base}/steps/{step_index}/parent"), + format!("parent step {parent} does not exist in this inference"), + ); + } + } + } + + for id in parents.keys().copied() { + let mut path = HashSet::new(); + let mut current = Some(id); + while let Some(step_id) = current { + if !path.insert(step_id) { + push( + diagnostics, + "TRACE_R3_STEP_CYCLE", + format!("{base}/steps"), + format!("provenance chain containing {step_id} is cyclic"), + ); + break; + } + current = parents.get(step_id).copied().flatten(); + } + } + + if let Some(segments) = inference.get("segments").and_then(Value::as_array) { + for (segment_index, segment) in segments.iter().enumerate() { + if let Some(step) = segment.get("step").and_then(Value::as_str) { + if !parents.contains_key(step) { + push( + diagnostics, + "TRACE_R2_STEP_NOT_FOUND", + format!("{base}/segments/{segment_index}/step"), + format!("segment provenance step {step} does not exist in this inference"), + ); + } + } + } + } +} + +fn reconstruct_segments( + segments: &[Value], + base: &str, + inline_threshold: usize, + content_mode: &str, + artifacts: &ArtifactIndex<'_>, + diagnostics: &mut Vec, +) -> Reconstructed { + let mut bytes = Vec::new(); + let mut unavailable = false; + let mut invalid = false; + let mut segment_ids = HashSet::new(); + + for (segment_index, segment) in segments.iter().enumerate() { + let segment_base = format!("{base}/segments/{segment_index}"); + if segment.get("i").and_then(Value::as_u64) != u64::try_from(segment_index).ok() { + push( + diagnostics, + "TRACE_R1_SEGMENT_INDEX_INVALID", + format!("{segment_base}/i"), + "segment index must equal its zero-based list position", + ); + } + if let Some(id) = segment.get("id").and_then(Value::as_str) { + if !segment_ids.insert(id) { + push( + diagnostics, + "TRACE_SEGMENT_ID_DUPLICATE", + format!("{segment_base}/id"), + format!("segment id {id} occurs more than once"), + ); + } + } + + let Some(content) = segment.get("content") else { + continue; + }; + match resolve_content( + content, + &format!("{segment_base}/content"), + inline_threshold, + content_mode, + artifacts, + diagnostics, + ) { + Reconstructed::Bytes(mut content_bytes) => bytes.append(&mut content_bytes), + Reconstructed::Unavailable => unavailable = true, + Reconstructed::Invalid => invalid = true, + } + } + + if invalid { + Reconstructed::Invalid + } else if unavailable { + Reconstructed::Unavailable + } else { + Reconstructed::Bytes(bytes) + } +} + +fn resolve_content( + content: &Value, + pointer: &str, + inline_threshold: usize, + content_mode: &str, + artifacts: &ArtifactIndex<'_>, + diagnostics: &mut Vec, +) -> Reconstructed { + match content.get("state").and_then(Value::as_str) { + Some("inline") => { + let text = content + .get("text") + .and_then(Value::as_str) + .unwrap_or_default(); + if text.len() > inline_threshold { + push( + diagnostics, + "TRACE_INLINE_CONTENT_EXCEEDS_THRESHOLD", + pointer, + "inline content exceeds the declared UTF-8 byte threshold", + ); + } + Reconstructed::Bytes(text.as_bytes().to_vec()) + } + Some("external") => { + let sha256 = content + .get("sha256") + .and_then(Value::as_str) + .unwrap_or_default(); + let declared_length = content + .get("length") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .unwrap_or(0); + + if let Some(text) = content + .pointer("/materialized/text") + .and_then(Value::as_str) + { + return validate_content_bytes( + text.as_bytes(), + sha256, + declared_length, + pointer, + diagnostics, + ); + } + + if content_mode == "materialized" { + push( + diagnostics, + "TRACE_R5_MATERIALIZATION_REQUIRED", + pointer, + "materialized export contains an unresolved external content reference", + ); + return Reconstructed::Invalid; + } + + if artifacts.invalid.contains(sha256) { + return Reconstructed::Invalid; + } + if let Some(artifact) = artifacts.valid.get(sha256) { + return validate_content_bytes( + artifact.content_utf8.as_bytes(), + sha256, + declared_length, + pointer, + diagnostics, + ); + } + Reconstructed::Unavailable + } + Some("withheld") => { + if content_mode == "materialized" { + push( + diagnostics, + "TRACE_R5_MATERIALIZATION_REQUIRED", + pointer, + "materialized export cannot contain withheld content", + ); + Reconstructed::Invalid + } else { + Reconstructed::Unavailable + } + } + _ => Reconstructed::Invalid, + } +} + +fn validate_content_bytes( + bytes: &[u8], + declared_sha256: &str, + declared_length: usize, + pointer: &str, + diagnostics: &mut Vec, +) -> Reconstructed { + let mut invalid = false; + if bytes.len() != declared_length { + push( + diagnostics, + "TRACE_CONTENT_LENGTH_MISMATCH", + format!("{pointer}/length"), + "content UTF-8 byte length differs from its declared length", + ); + invalid = true; + } + if sha256_hex(bytes) != declared_sha256 { + push( + diagnostics, + "TRACE_CONTENT_DIGEST_MISMATCH", + format!("{pointer}/sha256"), + "content bytes differ from their declared SHA-256 digest", + ); + invalid = true; + } + + if invalid { + Reconstructed::Invalid + } else { + Reconstructed::Bytes(bytes.to_vec()) + } +} + +fn validate_boundary( + event: &Value, + base: &str, + content_mode: &str, + artifacts: &ArtifactIndex<'_>, + diagnostics: &mut Vec, +) { + for field in ["sent", "received"] { + if let Some(content) = event.get(field) { + let _ = resolve_content( + content, + &format!("{base}/{field}"), + usize::MAX, + content_mode, + artifacts, + diagnostics, + ); + } + } +} + +fn validate_routing( + inference: &Value, + base: &str, + transcript: Option<&TranscriptInference>, + diagnostics: &mut Vec, +) { + let Some(routing) = inference.get("routing") else { + return; + }; + let outcome = routing + .get("outcome") + .and_then(Value::as_str) + .unwrap_or_default(); + let policy = routing + .get("policy") + .and_then(Value::as_str) + .unwrap_or_default(); + let bound = routing.get("bound").and_then(Value::as_str); + let attempts = routing + .get("attempts") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let filled = attempts + .iter() + .enumerate() + .filter(|(_, attempt)| attempt.get("outcome").and_then(Value::as_str) == Some("filled")) + .collect::>(); + + let successful_bound = if outcome == "invoked" && filled.len() == 1 { + filled[0].1.get("bound").and_then(Value::as_str) + } else { + None + }; + + if (outcome == "invoked" && filled.len() != 1) || (outcome == "refused" && !filled.is_empty()) { + push( + diagnostics, + "TRACE_ROUTING_ATTEMPT_INVALID", + format!("{base}/routing/attempts"), + "routing outcome and successful attempt count disagree", + ); + } + if outcome == "invoked" && bound != successful_bound { + push( + diagnostics, + "TRACE_ROUTING_BOUND_MISMATCH", + format!("{base}/routing/bound"), + "routing bound entity differs from the successful attempt", + ); + } + + if let Some(transcript) = transcript { + if transcript.routing.outcome != outcome { + push( + diagnostics, + "TRACE_PRODUCER_ROUTING_OUTCOME_MISMATCH", + format!("{base}/routing/outcome"), + "trace routing outcome differs from the producer transcript", + ); + } + if transcript.routing.bound.as_deref() != bound { + push( + diagnostics, + "TRACE_PRODUCER_BOUND_MISMATCH", + format!("{base}/routing/bound"), + "trace bound entity differs from the independently observed entity", + ); + } + } + + if let Some(fill) = routing.get("fill").and_then(Value::as_array) { + for (fill_index, entry) in fill.iter().enumerate() { + if entry.get("kind").and_then(Value::as_str) != Some("entity") { + continue; + } + let pointer = format!("{base}/routing/fill/{fill_index}"); + validate_entity_fill( + entry, + &pointer, + policy, + outcome, + bound, + attempts, + diagnostics, + ); + } + } + + validate_resolution(inference, base, outcome, successful_bound, diagnostics); +} + +#[allow(clippy::too_many_lines)] +fn validate_entity_fill( + fill: &Value, + pointer: &str, + routing_policy: &str, + routing_outcome: &str, + routing_bound: Option<&str>, + attempts: &[Value], + diagnostics: &mut Vec, +) { + let status = fill + .get("status") + .and_then(Value::as_str) + .unwrap_or_default(); + let requested = fill + .get("requested") + .and_then(Value::as_str) + .unwrap_or_default(); + let applied = fill.get("applied").and_then(Value::as_str); + let policy = fill + .get("policy") + .and_then(Value::as_str) + .unwrap_or_default(); + + if policy != routing_policy { + push( + diagnostics, + "TRACE_ENTITY_POLICY_MISMATCH", + format!("{pointer}/policy"), + "entity fill policy differs from the routing policy", + ); + } + + match status { + "kept" => { + if applied != Some(requested) { + push( + diagnostics, + "TRACE_ENTITY_KEPT_MISMATCH", + format!("{pointer}/applied"), + "kept status requires requested and applied canonical entities to match", + ); + } + if applied != routing_bound { + push( + diagnostics, + "TRACE_ROUTING_FILL_MISMATCH", + format!("{pointer}/applied"), + "entity fill differs from the routing bound entity", + ); + } + } + "fallback" => { + let step = fill + .get("route_step") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()); + let step_valid = step.is_some_and(|index| { + attempts.get(index).is_some_and(|attempt| { + attempt.get("outcome").and_then(Value::as_str) == Some("filled") + && attempt.get("bound").and_then(Value::as_str) == applied + }) && attempts[..index].iter().all(|attempt| { + matches!( + attempt.get("outcome").and_then(Value::as_str), + Some("failed" | "blocked") + ) + }) + }); + if !step_valid { + push( + diagnostics, + "TRACE_ENTITY_FALLBACK_STEP_INVALID", + format!("{pointer}/route_step"), + "fallback route_step does not identify the first successful legal attempt", + ); + } + if applied != routing_bound { + push( + diagnostics, + "TRACE_ROUTING_FILL_MISMATCH", + format!("{pointer}/applied"), + "fallback fill differs from the routing bound entity", + ); + } + } + "substituted" => { + let authority = fill + .get("deciding_authority") + .and_then(Value::as_str) + .unwrap_or_default(); + if policy != "best-effort" + || authority != "venue-operations" + || applied == Some(requested) + { + push( + diagnostics, + "TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED", + pointer, + "substitution requires explicit best-effort policy, a changed entity, and venue-operations authority", + ); + } + if applied != routing_bound { + push( + diagnostics, + "TRACE_ROUTING_FILL_MISMATCH", + format!("{pointer}/applied"), + "substitution fill differs from the routing bound entity", + ); + } + } + "refused" if routing_outcome != "refused" || applied.is_some() => push( + diagnostics, + "TRACE_ENTITY_REFUSAL_MISMATCH", + pointer, + "refused entity fill requires a refused routing outcome and no applied entity", + ), + _ => {} + } +} + +fn validate_resolution( + inference: &Value, + base: &str, + routing_outcome: &str, + successful_bound: Option<&str>, + diagnostics: &mut Vec, +) { + let bindings = inference + .pointer("/resolution/bindings") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + for (binding_index, binding) in bindings.iter().enumerate() { + if let Some(bound) = binding.get("bound").and_then(Value::as_str) { + if !is_pinned_canonical_entity(bound) { + push( + diagnostics, + "TRACE_RESOLUTION_BINDING_UNPINNED", + format!("{base}/resolution/bindings/{binding_index}/bound"), + "resolution binding is not a canonical pinned entity identifier", + ); + } + } + } + + if routing_outcome == "invoked" { + if let Some(successful_bound) = successful_bound { + if !bindings.iter().any(|binding| { + binding.get("bound").and_then(Value::as_str) == Some(successful_bound) + }) { + push( + diagnostics, + "TRACE_RESOLUTION_BOUND_MISSING", + format!("{base}/resolution/bindings"), + "resolution report does not contain the successfully invoked entity", + ); + } + } + } else if inference + .pointer("/resolution/refusals") + .and_then(Value::as_array) + .is_none_or(Vec::is_empty) + { + push( + diagnostics, + "TRACE_RESOLUTION_REFUSAL_MISSING", + format!("{base}/resolution/refusals"), + "refused routing outcome has no typed resolution refusal", + ); + } +} + +fn build_artifact_index<'a>( + transcript: Option<&'a ProducerTranscript>, + diagnostics: &mut Vec, +) -> ArtifactIndex<'a> { + let mut valid = HashMap::new(); + let mut invalid = HashSet::new(); + let Some(transcript) = transcript else { + return ArtifactIndex { valid, invalid }; + }; + + for (index, artifact) in transcript.external_artifacts.iter().enumerate() { + let mut artifact_valid = true; + if sha256_hex(artifact.content_utf8.as_bytes()) != artifact.sha256 { + push( + diagnostics, + "TRACE_CONTENT_DIGEST_MISMATCH", + format!("/transcript/external_artifacts/{index}/sha256"), + "external artifact bytes differ from their declared SHA-256 digest", + ); + artifact_valid = false; + } + if artifact.content_utf8.len() != artifact.length { + push( + diagnostics, + "TRACE_CONTENT_LENGTH_MISMATCH", + format!("/transcript/external_artifacts/{index}/length"), + "external artifact UTF-8 byte length differs from its declared length", + ); + artifact_valid = false; + } + if valid.contains_key(artifact.sha256.as_str()) + || invalid.contains(artifact.sha256.as_str()) + { + push( + diagnostics, + "TRACE_PRODUCER_ARTIFACT_DUPLICATE", + format!("/transcript/external_artifacts/{index}/sha256"), + "external artifact digest occurs more than once", + ); + artifact_valid = false; + } + + if artifact_valid { + valid.insert(artifact.sha256.as_str(), artifact); + } else { + invalid.insert(artifact.sha256.as_str()); + } + } + + ArtifactIndex { valid, invalid } +} + +fn is_pinned_canonical_entity(value: &str) -> bool { + let Some((namespace, version)) = value.rsplit_once('@') else { + return false; + }; + !version.is_empty() && namespace.contains(':') && namespace.contains('/') +} + +fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut output = String::with_capacity(64); + for byte in digest { + write!(&mut output, "{byte:02x}").expect("writing to a String cannot fail"); + } + output +} + +fn push( + diagnostics: &mut Vec, + code: &str, + pointer: impl Into, + message: impl Into, +) { + diagnostics.push(Diagnostic { + code: code.to_owned(), + pointer: pointer.into(), + message: message.into(), + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_kept_fixture_without_semantic_diagnostics() { + let trace: Value = + serde_json::from_str(include_str!("../../fixtures/trace-user-kept.json")) + .expect("valid fixture"); + assert!(validate_trace(&trace, None).is_empty()); + } + + #[test] + fn detects_request_byte_mismatch() { + let trace: Value = + serde_json::from_str(include_str!("../../fixtures/trace-user-kept.json")) + .expect("valid fixture"); + let transcript = ProducerTranscript { + format_version: "0.1-draft".to_owned(), + turn: "turn-example-001".to_owned(), + inferences: vec![TranscriptInference { + id: "inference-001".to_owned(), + actual_request_utf8: "different".to_owned(), + routing: TranscriptRouting { + outcome: "invoked".to_owned(), + bound: Some("model:example/atlas-4@2026-08-01".to_owned()), + }, + }], + boundaries: Vec::new(), + external_artifacts: Vec::new(), + }; + let diagnostics = validate_trace(&trace, Some(&transcript)); + assert!( + diagnostics + .iter() + .any(|item| item.code == "TRACE_R1_REQUEST_BYTES_MISMATCH") + ); + } +} diff --git a/conformance/transcript.schema.json b/conformance/transcript.schema.json new file mode 100644 index 0000000..093d681 --- /dev/null +++ b/conformance/transcript.schema.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/producer-transcript/0.1-draft", + "title": "PromptSyntax deterministic producer transcript", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "turn", "inferences", "boundaries", "external_artifacts"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "turn": { "type": "string", "minLength": 1 }, + "inferences": { + "type": "array", + "items": { "$ref": "#/$defs/inference" } + }, + "boundaries": { + "type": "array", + "items": { "$ref": "#/$defs/boundary" } + }, + "external_artifacts": { + "type": "array", + "items": { "$ref": "#/$defs/artifact" } + } + }, + "$defs": { + "inference": { + "type": "object", + "additionalProperties": false, + "required": ["id", "actual_request_utf8", "routing"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "actual_request_utf8": { "type": "string" }, + "routing": { + "type": "object", + "additionalProperties": false, + "required": ["outcome"], + "properties": { + "outcome": { "enum": ["invoked", "refused"] }, + "bound": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "if": { + "required": ["outcome"], + "properties": { "outcome": { "const": "invoked" } } + }, + "then": { "required": ["bound"] }, + "else": { "not": { "required": ["bound"] } } + } + ] + } + } + }, + "boundary": { + "type": "object", + "additionalProperties": false, + "required": ["id", "outcome"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "outcome": { "enum": ["succeeded", "failed", "blocked", "cancelled"] } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["sha256", "length", "content_utf8"], + "properties": { + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "length": { "type": "integer", "minimum": 0 }, + "content_utf8": { "type": "string" } + } + } + } +} diff --git a/decisions/2026-08-03-conformance-baseline-proposal.md b/decisions/2026-08-03-conformance-baseline-proposal.md index 963d543..43d30f6 100644 --- a/decisions/2026-08-03-conformance-baseline-proposal.md +++ b/decisions/2026-08-03-conformance-baseline-proposal.md @@ -1,7 +1,7 @@ # Decision proposal: conformance baseline and execution-receipt outcomes - **Date:** 2026-08-03 -- **Status:** Proposed for v0.3 +- **Status:** Proposed for v0.3; exercised by candidate suite `0.1.0-rc.1` - **Spec locations:** §4, §8, §10.5, §12, §16 OQ14 and OQ15 - **Decision requested:** How PromptSyntax freezes a test oracle and represents exact binding, fallback, substitution, and refusal without conflating them. @@ -203,6 +203,12 @@ typed Rust runner against PromptSyntax requirements. No Prompty runtime is requi ## Questions for PR review +Candidate `0.1.0-rc.1` uses the recommended answers below so executable review can proceed. +They remain provisional until this decision is accepted: substitution is legal only under +an explicitly selected best-effort policy; capacity substitution names +`venue-operations`; R1 producer checks compare independent UTF-8 request bytes without +normalization; and unknown fields are rejected outside namespaced `extensions` objects. + 1. Should `substituted` ever be legal under an explicitly selected `best-effort` entity policy, or should entity substitution always require pre-execution confirmation? 2. Should the user-tier receipt expose the deciding authority for capacity substitution as diff --git a/examples/prompt-trace-0.3-draft.example.json b/examples/prompt-trace-0.3-draft.example.json index 6abc7a4..d354fa9 100644 --- a/examples/prompt-trace-0.3-draft.example.json +++ b/examples/prompt-trace-0.3-draft.example.json @@ -3,6 +3,7 @@ "turn": "turn-example-001", "tier": "user", "mode": "executed", + "content_mode": "tiered", "venue": { "id": "venue:example", "capability_document": "https://venue.example/ps-capabilities/0.3.json", diff --git a/schemas/prompt-trace-0.3-draft.schema.json b/schemas/prompt-trace-0.3-draft.schema.json index 6690afc..b7e1bcc 100644 --- a/schemas/prompt-trace-0.3-draft.schema.json +++ b/schemas/prompt-trace-0.3-draft.schema.json @@ -5,12 +5,13 @@ "description": "Non-normative candidate schema used to resolve the v0.2 conformance audit. It must not be used for an unqualified conformance claim.", "type": "object", "additionalProperties": false, - "required": ["ps_trace", "turn", "tier", "mode", "venue", "coverage", "events"], + "required": ["ps_trace", "turn", "tier", "mode", "content_mode", "venue", "coverage", "events"], "properties": { "ps_trace": { "const": "0.3-draft" }, "turn": { "type": "string", "minLength": 1 }, "tier": { "enum": ["user", "developer", "operator", "oversight"] }, "mode": { "enum": ["dry-run", "executed"] }, + "content_mode": { "enum": ["tiered", "materialized"] }, "venue": { "$ref": "#/$defs/venue" }, "coverage": { "$ref": "#/$defs/coverage" }, "integrity": { "$ref": "#/$defs/integrity" }, @@ -18,10 +19,7 @@ "type": "array", "minItems": 1, "items": { - "oneOf": [ - { "$ref": "#/$defs/inference" }, - { "$ref": "#/$defs/boundary_event" } - ] + "oneOf": [{ "$ref": "#/$defs/inference" }, { "$ref": "#/$defs/boundary_event" }] } }, "extensions": { "$ref": "#/$defs/extensions" } @@ -62,10 +60,7 @@ "kind": { "enum": ["user-initiated", "internal-continuation"] }, "routing": { "$ref": "#/$defs/routing_record" }, "segments": { - "oneOf": [ - { "$ref": "#/$defs/segment_list" }, - { "$ref": "#/$defs/segments_omitted" } - ] + "oneOf": [{ "$ref": "#/$defs/segment_list" }, { "$ref": "#/$defs/segments_omitted" }] }, "steps": { "type": "array", From 9d82fc7b5742d5496ce438cd836a09dc2b336226 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 01:30:29 +0700 Subject: [PATCH 3/5] ci: pin parser-compatible toolchains --- .github/workflows/conformance.yml | 2 +- conformance/implementations.lock.json | 2 ++ conformance/implementations.schema.json | 11 ++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 4af8308..e3edd1c 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -78,7 +78,7 @@ jobs: - name: Install stable Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.85.0 + toolchain: 1.97.0 - name: Install Bun uses: oven-sh/setup-bun@v2 diff --git a/conformance/implementations.lock.json b/conformance/implementations.lock.json index 19f3a61..bd7c92d 100644 --- a/conformance/implementations.lock.json +++ b/conformance/implementations.lock.json @@ -7,6 +7,7 @@ "commit": "c64072f618ffdb53cc389fc40dfe0370c4cf05be", "adapter": "tests/conformance.rs", "fixture": "tests/conformance.json", + "runtime": { "name": "rust", "version": "1.97.0" }, "command": ["cargo", "test"] }, { @@ -15,6 +16,7 @@ "commit": "b22acfac066c0ea6eec7d042d69040c695ef5422", "adapter": "test/parser.test.ts", "fixture": "test/conformance.json", + "runtime": { "name": "bun", "version": "1.3.14" }, "command": ["bun", "test"] } ] diff --git a/conformance/implementations.schema.json b/conformance/implementations.schema.json index 3901232..e7690ec 100644 --- a/conformance/implementations.schema.json +++ b/conformance/implementations.schema.json @@ -13,13 +13,22 @@ "items": { "type": "object", "additionalProperties": false, - "required": ["id", "repository", "commit", "adapter", "fixture", "command"], + "required": ["id", "repository", "commit", "adapter", "fixture", "runtime", "command"], "properties": { "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, "repository": { "type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" }, "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, "adapter": { "type": "string", "minLength": 1 }, "fixture": { "type": "string", "minLength": 1 }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version"], + "properties": { + "name": { "enum": ["rust", "bun"] }, + "version": { "type": "string", "minLength": 1 } + } + }, "command": { "type": "array", "minItems": 1, From 646a04c10bf3dd13b867f10a2965c74ccfa76401 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 03:07:35 +0700 Subject: [PATCH 4/5] feat: add cross-language conformance adapters --- .github/workflows/conformance.yml | 98 ++- README.md | 15 +- conformance/README.md | 30 +- conformance/adapters/README.md | 44 +- conformance/cases/core-parser.json | 797 ++++++++++++++++++ conformance/cases/trace-producer.json | 64 ++ conformance/core-adapter-result.schema.json | 289 +++++++ conformance/core-cases.schema.json | 3 +- .../core-differential-input.schema.json | 33 + .../core-differential-report.schema.json | 67 ++ conformance/diagnostic-registry.json | 240 ++++++ conformance/diagnostic-registry.schema.json | 4 +- .../producer/input-boundary-failed.json | 63 ++ .../producer/input-boundary-succeeded.json | 63 ++ .../fixtures/producer/input-fallback.json | 56 ++ conformance/fixtures/producer/input-kept.json | 56 ++ .../producer/input-missing-refusal.json | 41 + .../producer/input-multiple-filled.json | 54 ++ .../producer/input-refusal-conflict.json | 47 ++ .../fixtures/producer/input-refused.json | 49 ++ .../producer/input-strict-substitution.json | 42 + .../fixtures/producer/input-substituted.json | 45 + .../producer/transcript-boundary-failed.json | 16 + .../transcript-boundary-succeeded.json | 16 + .../producer/transcript-fallback.json | 16 + .../fixtures/producer/transcript-kept.json | 16 + .../fixtures/producer/transcript-refused.json | 13 + .../producer/transcript-substituted.json | 16 + conformance/implementations.lock.json | 29 +- conformance/implementations.schema.json | 24 +- conformance/manifest.json | 12 +- conformance/manifest.schema.json | 24 +- conformance/releases/0.1.0-rc.1.md | 16 +- .../reports/core-differential-0.1.0-rc.1.json | 19 + ...ore-generated-differential-0.1.0-rc.1.json | 23 + .../reports/trace-producer-rs-0.1.0-rc.1.json | 664 +++++++++++++++ .../reports/trace-producer-ts-0.1.0-rc.1.json | 664 +++++++++++++++ conformance/runner/README.md | 35 +- conformance/runner/src/core.rs | 740 ++++++++++++++++ conformance/runner/src/generator.rs | 330 ++++++++ conformance/runner/src/lib.rs | 58 ++ conformance/runner/src/main.rs | 221 ++++- conformance/runner/src/producer.rs | 305 +++++++ conformance/runner/src/suite.rs | 138 ++- conformance/trace-producer-cases.schema.json | 66 ++ conformance/trace-producer-input.schema.json | 375 ++++++++ 46 files changed, 5996 insertions(+), 40 deletions(-) create mode 100644 conformance/cases/trace-producer.json create mode 100644 conformance/core-adapter-result.schema.json create mode 100644 conformance/core-differential-input.schema.json create mode 100644 conformance/core-differential-report.schema.json create mode 100644 conformance/fixtures/producer/input-boundary-failed.json create mode 100644 conformance/fixtures/producer/input-boundary-succeeded.json create mode 100644 conformance/fixtures/producer/input-fallback.json create mode 100644 conformance/fixtures/producer/input-kept.json create mode 100644 conformance/fixtures/producer/input-missing-refusal.json create mode 100644 conformance/fixtures/producer/input-multiple-filled.json create mode 100644 conformance/fixtures/producer/input-refusal-conflict.json create mode 100644 conformance/fixtures/producer/input-refused.json create mode 100644 conformance/fixtures/producer/input-strict-substitution.json create mode 100644 conformance/fixtures/producer/input-substituted.json create mode 100644 conformance/fixtures/producer/transcript-boundary-failed.json create mode 100644 conformance/fixtures/producer/transcript-boundary-succeeded.json create mode 100644 conformance/fixtures/producer/transcript-fallback.json create mode 100644 conformance/fixtures/producer/transcript-kept.json create mode 100644 conformance/fixtures/producer/transcript-refused.json create mode 100644 conformance/fixtures/producer/transcript-substituted.json create mode 100644 conformance/reports/core-differential-0.1.0-rc.1.json create mode 100644 conformance/reports/core-generated-differential-0.1.0-rc.1.json create mode 100644 conformance/reports/trace-producer-rs-0.1.0-rc.1.json create mode 100644 conformance/reports/trace-producer-ts-0.1.0-rc.1.json create mode 100644 conformance/runner/src/core.rs create mode 100644 conformance/runner/src/generator.rs create mode 100644 conformance/runner/src/producer.rs create mode 100644 conformance/trace-producer-cases.schema.json create mode 100644 conformance/trace-producer-input.schema.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index e3edd1c..2dd1c83 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -53,7 +53,7 @@ jobs: cmp conformance/reports/0.1.0-rc.1.json /tmp/promptsyntax-conformance-report.json cross-language-core: - name: Rust and TypeScript Core adapters + name: Cross-language Core and Trace adapters runs-on: ubuntu-latest steps: - name: Checkout standard and canonical cases @@ -65,14 +65,14 @@ jobs: uses: actions/checkout@v4 with: repository: pathscale/PromptSyntax-rs - ref: c64072f618ffdb53cc389fc40dfe0370c4cf05be + ref: 2789fac8acb810279ece887f46c025ef987ce0bf path: implementations/promptsyntax-rs - name: Checkout pinned TypeScript implementation uses: actions/checkout@v4 with: repository: pathscale/PromptSyntax-ts - ref: b22acfac066c0ea6eec7d042d69040c695ef5422 + ref: 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 path: implementations/promptsyntax-ts - name: Install stable Rust @@ -85,17 +85,97 @@ jobs: with: bun-version: 1.3.14 - - name: Run Rust adapter against canonical cases - run: | - cp standard/conformance/cases/core-parser.json implementations/promptsyntax-rs/tests/conformance.json - cargo test --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml - - name: Install TypeScript implementation dependencies run: bun install --frozen-lockfile working-directory: implementations/promptsyntax-ts - - name: Run TypeScript adapter against canonical cases + - name: Run Rust implementation checks against canonical cases + run: | + cp standard/conformance/cases/core-parser.json implementations/promptsyntax-rs/tests/conformance.json + cargo fmt --manifest-path implementations/promptsyntax-rs/Cargo.toml -- --check + cargo test --locked --all-targets --manifest-path implementations/promptsyntax-rs/Cargo.toml + cargo clippy --locked --all-targets --manifest-path implementations/promptsyntax-rs/Cargo.toml -- -D warnings + + - name: Run TypeScript implementation checks against canonical cases run: | cp ../../standard/conformance/cases/core-parser.json test/conformance.json bun test + bun run typecheck + bun run lint + bun run build working-directory: implementations/promptsyntax-ts + + - name: Produce canonical normalized Core results + run: | + cargo run --quiet --release --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml --bin ps-core-adapter -- \ + standard/conformance/cases/core-parser.json 2789fac8acb810279ece887f46c025ef987ce0bf \ + > /tmp/promptsyntax-rs-core.json + bun implementations/promptsyntax-ts/src/bin/core-adapter.ts \ + standard/conformance/cases/core-parser.json 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 \ + > /tmp/promptsyntax-ts-core.json + + - name: Validate and compare canonical Core results + run: | + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/core-adapter-result.schema.json /tmp/promptsyntax-rs-core.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/core-adapter-result.schema.json /tmp/promptsyntax-ts-core.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + compare-core-adapters standard/conformance/cases/core-parser.json \ + /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json \ + > /tmp/core-differential.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/core-differential-report.schema.json /tmp/core-differential.json + cmp standard/conformance/reports/core-differential-0.1.0-rc.1.json /tmp/core-differential.json + + - name: Generate deterministic 100,000-case differential input + run: | + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + generate-core-differential 100000 20270803 > /tmp/core-generated.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/core-differential-input.schema.json /tmp/core-generated.json + + - name: Stream generated Core results from both implementations + run: | + cargo run --quiet --release --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml --bin ps-core-adapter -- \ + /tmp/core-generated.json 2789fac8acb810279ece887f46c025ef987ce0bf --jsonl \ + > /tmp/promptsyntax-rs-generated.jsonl + bun implementations/promptsyntax-ts/src/bin/core-adapter.ts \ + /tmp/core-generated.json 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 --jsonl \ + > /tmp/promptsyntax-ts-generated.jsonl + + - name: Compare all 100,000 generated normalized outputs + run: | + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + compare-core-streams 100000 20270803 \ + /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl \ + > /tmp/core-generated-differential.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/core-differential-report.schema.json \ + /tmp/core-generated-differential.json + cmp standard/conformance/reports/core-generated-differential-0.1.0-rc.1.json \ + /tmp/core-generated-differential.json + + - name: Run both real Trace producers + run: | + cargo build --quiet --release --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml \ + --bin ps-trace-producer + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + run-trace-producer-adapter standard promptsyntax-rs 0.1.0 \ + 2789fac8acb810279ece887f46c025ef987ce0bf \ + implementations/promptsyntax-rs/target/release/ps-trace-producer \ + > /tmp/trace-producer-rs.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + run-trace-producer-adapter standard promptsyntax-ts 0.1.0 \ + 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 \ + bun implementations/promptsyntax-ts/src/bin/trace-producer.ts \ + > /tmp/trace-producer-ts.json + + - name: Validate and freeze Trace producer reports + run: | + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/adapter-result.schema.json /tmp/trace-producer-rs.json + cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \ + validate-instance standard/conformance/adapter-result.schema.json /tmp/trace-producer-ts.json + cmp standard/conformance/reports/trace-producer-rs-0.1.0-rc.1.json /tmp/trace-producer-rs.json + cmp standard/conformance/reports/trace-producer-ts-0.1.0-rc.1.json /tmp/trace-producer-ts.json diff --git a/README.md b/README.md index ebf50a1..67434b0 100644 --- a/README.md +++ b/README.md @@ -82,11 +82,22 @@ cargo test --manifest-path conformance/runner/Cargo.toml cargo run --manifest-path conformance/runner/Cargo.toml -- \ check-requirements conformance/requirements.json cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-adapters conformance/cases/core-parser.json \ + /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json +cargo run --release --manifest-path conformance/runner/Cargo.toml -- \ + generate-core-differential 100000 20270803 > /tmp/core-generated.json +cargo run --release --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-streams 100000 20270803 \ + /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl ``` Candidate `0.1.0-rc.1` currently executes 34 Trace document and producer cases with full -coverage of its explicitly named 22-requirement profile. Candidate status is not a released -or unqualified implementation-conformance claim. +coverage of its explicitly named 22-requirement profile. It also carries 101 Core parser +vectors, compares full normalized Rust and TypeScript outputs, runs a deterministic +100,000-input grammar differential, and runs ten real Trace producer cases against +independent transcripts. Candidate status is not a released or unqualified +implementation-conformance claim. ## Contributing diff --git a/conformance/README.md b/conformance/README.md index dda3d58..bebf2e0 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -9,7 +9,12 @@ are the oracle. Implementations consume the corpus but do not define its expecte Version `0.1.0-rc.1` is a candidate suite against the specification draft at commit `7456634817f4eb68c8909a58b476505e6fd7c063`. It executes 34 document and producer cases across four Trace families and covers all 22 requirements in its named candidate profile. -It is not a released conformance suite and must not be used for an unqualified claim. +It also owns 101 Core parser vectors and a full normalized-output differential protocol for +the Rust and TypeScript implementations. A deterministic grammar generator adds a +100,000-input differential lane without treating either parser as the expected-answer +oracle. A separate ten-case executable adapter corpus tests real Trace producers in both +languages against independent transcripts. It is not a released conformance suite and must +not be used for an unqualified claim. The cases freeze provisional v0.3 decisions for review without pretending those decisions are already normative. The machine-readable inventory retains blocked, proposed, and open @@ -34,6 +39,21 @@ cargo run --manifest-path conformance/runner/Cargo.toml -- \ check-requirements conformance/requirements.json cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . + +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-adapters conformance/cases/core-parser.json \ + /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json + +cargo run --release --manifest-path conformance/runner/Cargo.toml -- \ + generate-core-differential 100000 20270803 > /tmp/core-generated.json + +cargo run --release --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-streams 100000 20270803 \ + /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl + +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + run-trace-producer-adapter . promptsyntax-rs 0.1.0 \ + /path/to/ps-trace-producer ``` The command emits the deterministic report stored in @@ -44,7 +64,13 @@ The command emits the deterministic report stored in - `profiles/` freezes the named scope and candidate semantic decisions. - `families/` contains language-independent cases and expected diagnostics. - `fixtures/` contains reusable Trace and deterministic transcript inputs. -- `cases/core-parser.json` is the single Core fixture consumed by both parser adapters. +- `cases/core-parser.json` is the single 101-case Core fixture consumed by both adapters. +- `core-differential-input.schema.json` freezes the generator-to-adapter input shape. +- `core-adapter-result.schema.json` freezes the normalized cross-language output shape. +- `core-differential-report.schema.json` freezes the independent comparison report shape. +- `cases/trace-producer.json` lists six valid and four rejection cases for real producers. +- `trace-producer-input.schema.json` defines deterministic facts without expected output. +- `trace-producer-cases.schema.json` freezes the external producer adapter protocol. - `reports/` contains deterministic, machine-readable evidence snapshots. - `runner/` contains the independent stable-Rust checker. - `adapters/` defines the implementation boundary and forbids embedded expected answers. diff --git a/conformance/adapters/README.md b/conformance/adapters/README.md index 0e7e3da..e875407 100644 --- a/conformance/adapters/README.md +++ b/conformance/adapters/README.md @@ -4,11 +4,39 @@ The suite owns expected answers. An implementation adapter only converts a canon into an implementation call and converts the result into the language-neutral adapter result shape. Adapters must not contain alternate expectations or silently skip cases. -The current Rust and TypeScript parser tests already implement the Core adapter contract: -they consume the same fields in `conformance/cases/core-parser.json` and compare round-trip -text, data-plane text, directive kinds, and diagnostic codes. CI injects that one canonical -file into both implementation checkouts before running their native test commands. - -A Trace producer adapter receives deterministic transcript facts and emits a Prompt Trace. -The repository-owned runner then evaluates the emitted trace. Provider APIs and live model -calls are outside the normative adapter protocol because they are not deterministic. +The Rust and TypeScript Core adapters consume the same 101 specification-owned cases in +`conformance/cases/core-parser.json`. Each adapter checks round-trip text, data-plane text, +directive kinds, and diagnostic codes against the canonical expectations. It also emits a +normalized record containing the complete segment tree, directive AST, UTF-8 byte spans, +source slices, and parser diagnostics. + +The repository-owned runner compares those normalized records case by case. A mismatch is +a differential finding even when both implementations satisfy the coarser expected fields. +Agreement between two implementations is compatibility evidence, not a substitute for the +specification-owned expectations and not an independent certification claim. + +Adapters report canonical UTF-8 byte offsets. The TypeScript adapter converts its native +UTF-16 editor offsets at the boundary, while the Rust parser already exposes byte offsets. +Adapter code contains normalization only. Expected answers remain in the standard corpus. + +For the generated differential lane, adapters accept cases that intentionally omit expected +answers and emit one normalized result per JSON line. The repository-owned generator uses +32 syntax families, seed `20270803`, and an exact 100,000-case inventory. The streaming +runner compares one result pair at a time and rejects missing, reordered, duplicated, or +extra generated case IDs. + +A Trace producer adapter receives deterministic execution facts and emits a Prompt Trace. +The repository-owned runner separately evaluates it against an independent transcript. +Provider APIs and live model calls are outside the normative adapter protocol because they +are not deterministic. + +The current adapter corpus has ten cases. Six valid cases cover kept, authored fallback, +best-effort substitution, refusal, failed boundary calls, and successful boundary calls. +Four negative cases require typed rejection of strict substitution, multiple filled +attempts, missing refusal facts, and a refusal that conflicts with a filled attempt. + +The implementation command receives only `trace-producer-input.schema.json` facts. The +runner separately loads `transcript.schema.json` evidence, validates the emitted Trace +structurally and semantically, and checks exact request bytes, actual bound entity, routing +outcome, and boundary outcome. This separation prevents an adapter from copying expected +answers into its output. diff --git a/conformance/cases/core-parser.json b/conformance/cases/core-parser.json index 4b99865..809c51c 100644 --- a/conformance/cases/core-parser.json +++ b/conformance/cases/core-parser.json @@ -1,5 +1,6 @@ [ { + "id": "core-plain-text", "name": "plain text", "source": "Summarize this.", "options": {}, @@ -8,6 +9,7 @@ "diagnostics": [] }, { + "id": "core-qualified-reference", "name": "qualified reference", "source": "Read @file:q3-report.md", "options": {}, @@ -16,6 +18,7 @@ "diagnostics": [] }, { + "id": "core-declared-bare-entity-action", "name": "declared bare entity and action", "source": "@opus Summarize /concise", "options": { "entities": ["opus"], "actions": ["concise"] }, @@ -24,6 +27,7 @@ "diagnostics": [] }, { + "id": "core-unknown-bare-inert", "name": "unknown bare tokens stay inert", "source": "@unknown and /usr/bin", "options": {}, @@ -32,6 +36,7 @@ "diagnostics": [] }, { + "id": "core-email-escapes-inert", "name": "email and escapes stay inert", "source": "mail a@b.com and \\@model:x", "options": {}, @@ -40,6 +45,7 @@ "diagnostics": [] }, { + "id": "core-fenced-syntax-inert", "name": "fenced syntax stays inert", "source": "```ps\n@model:openai/gpt-5.6\n```", "options": {}, @@ -48,6 +54,7 @@ "diagnostics": [] }, { + "id": "core-full-width-unicode", "name": "full width sigils and Unicode identifiers", "source": "＠file:របាយការណ៍.md ／បកប្រែ", "options": { "actions": ["បកប្រែ"] }, @@ -56,6 +63,7 @@ "diagnostics": [] }, { + "id": "core-budgeted-fallback-route", "name": "budgeted fallback route", "source": "@opus! limit(wall_time: 5s) else @model:openai/gpt-5.6@2026-07-01 else ask\nDo it", "options": { "entities": ["opus"] }, @@ -64,6 +72,7 @@ "diagnostics": [] }, { + "id": "core-normal-span-inner-data", "name": "normal span keeps authored inner data", "source": "Before use @file:terms.md here after", "options": {}, @@ -72,6 +81,7 @@ "diagnostics": [] }, { + "id": "core-declared-authoring-segment", "name": "declared AgencyZero authoring segment", "source": "Before\n\nAfter", "options": { "authoring_namespaces": ["agency"] }, @@ -80,6 +90,7 @@ "diagnostics": [] }, { + "id": "core-undeclared-authoring-inert", "name": "undeclared authoring segment stays inert", "source": "", "options": {}, @@ -88,6 +99,7 @@ "diagnostics": ["UNCLOSED_SPAN"] }, { + "id": "core-strict-frontmatter", "name": "strict frontmatter", "source": "---ps\nversion: \"0.2\"\n---\nSummarize.", "options": {}, @@ -96,6 +108,7 @@ "diagnostics": [] }, { + "id": "core-bidi-rejects-partial", "name": "bidi control rejects partial parse", "source": "@model:openai/gpt\u202e-5", "options": {}, @@ -104,11 +117,795 @@ "diagnostics": ["BIDI_CONTROL"] }, { + "id": "core-malformed-qualified-reference", "name": "malformed qualified reference", "source": "Use @model: now", "options": {}, "data_plane": "Use @model: now", "directives": [], "diagnostics": ["SYNTAX_INVALID"] + }, + { + "id": "core-empty-input", + "name": "empty input", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-non-ps-tag-inert", + "name": "directive-shaped content in another tag stays inert", + "source": "text", + "options": {}, + "data_plane": "text", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-ps-prefix-tag-inert", + "name": "ps prefix tag is not the ps namespace", + "source": "text", + "options": {}, + "data_plane": "text", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-non-ps-self-closing-tag-inert", + "name": "directive-shaped attribute in a self-closing tag stays inert", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-html-comment-inert", + "name": "directive-shaped content in an HTML comment stays inert", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-processing-instruction-inert", + "name": "directive-shaped content in a processing instruction stays inert", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-doctype-inert", + "name": "directive-shaped content in a declaration stays inert", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-latin-attached-boundary", + "name": "Latin word attached reference stays inert", + "source": "prefix@model:atlas/atlas-4", + "options": {}, + "data_plane": "prefix@model:atlas/atlas-4", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-punctuation-boundary", + "name": "punctuation opens a point island boundary", + "source": "(@model:atlas/atlas-4)", + "options": {}, + "data_plane": "()", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-khmer-no-spacing-boundary", + "name": "Khmer prose without a space permits a point island", + "source": "សូម@model:atlas/atlas-4", + "options": {}, + "data_plane": "សូម", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-cjk-no-spacing-boundary", + "name": "CJK prose without a space permits a point island", + "source": "要約@model:atlas/atlas-4", + "options": {}, + "data_plane": "要約", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-full-width-escapes-inert", + "name": "escaped full-width sigils stay inert", + "source": "\\＠model:atlas/atlas-4 \\／translate", + "options": { "actions": ["translate"] }, + "data_plane": "\\＠model:atlas/atlas-4 \\／translate", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-tilde-fence-inert", + "name": "tilde fenced syntax stays inert", + "source": "~~~ps\n@model:atlas/atlas-4\n~~~", + "options": {}, + "data_plane": "~~~ps\n@model:atlas/atlas-4\n~~~", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-long-fence-inert", + "name": "longer fence ignores a shorter marker inside", + "source": "````ps\n@model:atlas/atlas-4\n```\n````", + "options": {}, + "data_plane": "````ps\n@model:atlas/atlas-4\n```\n````", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-indented-fence-inert", + "name": "three-space indented fence stays inert", + "source": " ```ps\n@model:atlas/atlas-4\n ```", + "options": {}, + "data_plane": " ```ps\n@model:atlas/atlas-4\n ```", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-unclosed-fence-inert", + "name": "unclosed fence protects through end of input", + "source": "```ps\n@model:atlas/atlas-4", + "options": {}, + "data_plane": "```ps\n@model:atlas/atlas-4", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-inline-backticks-not-fence", + "name": "inline backticks do not create a fenced block", + "source": "text ``` @model:atlas/atlas-4", + "options": {}, + "data_plane": "text ``` ", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-reference-scalar-arguments", + "name": "reference parses every scalar category", + "source": "@tool:search(query: \"atlas\", count: 3, enabled: true, cache: null, mode: fast)", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-escaped-string-arguments", + "name": "quoted arguments decode escapes", + "source": "@tool:search(query: \"Fix \\\"quoted\\\", text\", line: \"a\\nb\")", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-single-quoted-arguments", + "name": "single-quoted arguments decode matching quote escapes", + "source": "@tool:search(query: 'it\\'s ready', mode: fast)", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-strict-versioned-reference", + "name": "version and strict marker are preserved", + "source": "@model:atlas/atlas-4@2026-08-01!", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-nfc-qualified-reference", + "name": "qualified reference identifier is normalized to NFC", + "source": "@model:cafe\u0301", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-nfc-bare-resolution", + "name": "bare reference resolution uses NFC", + "source": "@cafe\u0301", + "options": { "entities": ["café"] }, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-route-fail-terminal", + "name": "route accepts an explicit fail terminal", + "source": "@model:atlas/atlas-4 else @model:atlas/atlas-mini else fail", + "options": {}, + "data_plane": "", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-route-without-terminal", + "name": "route may end after a fallback step", + "source": "@model:atlas/atlas-4 else @model:atlas/atlas-mini", + "options": {}, + "data_plane": "", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-route-case-insensitive-keywords", + "name": "route keywords accept ASCII case aliases", + "source": "@model:atlas/atlas-4 LIMIT(wall_time: 5s) ELSE ASK", + "options": {}, + "data_plane": "", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-route-fallback-limits", + "name": "each route step carries its own limits", + "source": "@model:atlas/atlas-4 limit(wall_time: 5s) else @model:atlas/atlas-mini limit(cost: $0.02) else fail", + "options": {}, + "data_plane": "", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-route-unknown-fallback-inert", + "name": "unresolvable bare fallback does not join a route", + "source": "@model:atlas/atlas-4 else @unknown", + "options": {}, + "data_plane": " else @unknown", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-route-incomplete-after-limit", + "name": "incomplete fallback text remains in the data plane", + "source": "@model:atlas/atlas-4 limit(wall_time: 5s) else", + "options": {}, + "data_plane": " else", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-empty-frontmatter", + "name": "empty strict frontmatter without final newline", + "source": "---ps\n---", + "options": {}, + "data_plane": "", + "directives": ["frontmatter"], + "diagnostics": [] + }, + { + "id": "core-windows-frontmatter", + "name": "strict frontmatter preserves Windows line endings", + "source": "---ps\r\nversion: \"0.2\"\r\n---\r\nBody", + "options": {}, + "data_plane": "Body", + "directives": ["frontmatter"], + "diagnostics": [] + }, + { + "id": "core-unclosed-frontmatter", + "name": "unclosed strict frontmatter fails closed", + "source": "---ps\nversion: \"0.2\"\nBody", + "options": {}, + "data_plane": "---ps\nversion: \"0.2\"\nBody", + "directives": [], + "diagnostics": ["UNCLOSED_FRONTMATTER"] + }, + { + "id": "core-frontmatter-not-document-start", + "name": "frontmatter marker outside document start stays content", + "source": "Preamble\n---ps\nversion: \"0.2\"\n---\nBody", + "options": {}, + "data_plane": "Preamble\n---ps\nversion: \"0.2\"\n---\nBody", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-frontmatter-opener-needs-newline", + "name": "frontmatter opener without newline stays content", + "source": "---ps", + "options": {}, + "data_plane": "---ps", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-span-attributes", + "name": "span parses canonical quoted attributes", + "source": "Summarize", + "options": {}, + "data_plane": "Summarize", + "directives": ["span"], + "diagnostics": [] + }, + { + "id": "core-span-mixed-header", + "name": "span header carries references and attributes in source order", + "source": "Use it", + "options": {}, + "data_plane": "Use it", + "directives": ["span"], + "diagnostics": [] + }, + { + "id": "core-nested-spans", + "name": "nested spans preserve inner prose and directive order", + "source": "Outer Inner @file:term.md end", + "options": {}, + "data_plane": "Outer Inner end", + "directives": ["span", "span", "reference"], + "diagnostics": [] + }, + { + "id": "core-fence-inside-span", + "name": "fenced closing tag text does not close a span", + "source": "\n```\n @model:atlas/atlas-4\n```\nAfter", + "options": {}, + "data_plane": "\n```\n @model:atlas/atlas-4\n```\nAfter", + "directives": ["span"], + "diagnostics": [] + }, + { + "id": "core-unclosed-span", + "name": "unclosed span stays content and reports a diagnostic", + "source": "Before body", + "options": {}, + "data_plane": "Before body", + "directives": [], + "diagnostics": ["UNCLOSED_SPAN"] + }, + { + "id": "core-malformed-declared-authoring", + "name": "malformed declared authoring segment fails closed as control", + "source": "", + "options": { "authoring_namespaces": ["agency"] }, + "data_plane": "", + "directives": ["invalid_authoring_segment"], + "diagnostics": ["SYNTAX_INVALID"] + }, + { + "id": "core-bidi-declared-authoring", + "name": "bidi control in a declared authoring segment fails closed as control", + "source": "", + "options": { "authoring_namespaces": ["agency"] }, + "data_plane": "", + "directives": ["invalid_authoring_segment"], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-bidi-action", + "name": "bidi control neutralizes an action island", + "source": "/trans\u202elate", + "options": { "actions": ["translate"] }, + "data_plane": "/trans\u202elate", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-malformed-reference-arguments", + "name": "malformed reference arguments fail closed", + "source": "@tool:search(query)", + "options": {}, + "data_plane": "@tool:search(query)", + "directives": [], + "diagnostics": ["SYNTAX_INVALID"] + }, + { + "id": "core-empty-reference-arguments", + "name": "empty argument list is accepted", + "source": "@tool:search()", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-trailing-argument-comma", + "name": "trailing argument comma is accepted", + "source": "@tool:search(query: \"atlas\",)", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-budget-number-units", + "name": "canonical budget units remain typed numeric scalars", + "source": "@model:atlas/atlas-4 limit(wall_time: 250ms, output: 800tok, cost: $0.02)", + "options": {}, + "data_plane": "", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-dotted-slashed-identifiers", + "name": "dotted and slashed identifier segments are preserved", + "source": "@vendor:family/model.variant-v2", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-emoji-boundary", + "name": "emoji before a sigil permits a point island", + "source": "✅@model:atlas/atlas-4", + "options": {}, + "data_plane": "✅", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-multiple-directive-order", + "name": "mixed directives remain in authored order", + "source": "@model:atlas/atlas-4 Write /translate(to: \"km\") using @file:notes.md", + "options": { "actions": ["translate"] }, + "data_plane": " Write using ", + "directives": ["reference", "action", "reference"], + "diagnostics": [] + }, + { + "id": "core-byte-exact-mixed-line-endings", + "name": "mixed line endings and Unicode round trip byte-exactly", + "source": "កម្ពុជា\r\n@model:atlas/atlas-4\n終わり", + "options": {}, + "data_plane": "កម្ពុជា\r\n\n終わり", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-reference-surrounded-by-tabs", + "name": "tabs around a qualified reference are preserved", + "source": "before\t@model:atlas/atlas-4\tafter", + "options": {}, + "data_plane": "before\t\tafter", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-punctuation-separated-references", + "name": "punctuation-separated references preserve authored order", + "source": "@file:first.md,@file:second.md", + "options": {}, + "data_plane": ",", + "directives": ["reference", "reference"], + "diagnostics": [] + }, + { + "id": "core-declared-unicode-bare-entity", + "name": "declared Unicode bare entity resolves", + "source": "@អាត្លាស", + "options": { "entities": ["អាត្លាស"] }, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-action-scalar-arguments", + "name": "action parses every scalar category", + "source": "/render(title: \"Atlas\", count: 2, enabled: false, cache: null, mode: fast)", + "options": { "actions": ["render"] }, + "data_plane": "", + "directives": ["action"], + "diagnostics": [] + }, + { + "id": "core-full-width-action-arguments", + "name": "full width action sigil accepts arguments", + "source": "／បកប្រែ(to: \"km\")", + "options": { "actions": ["បកប្រែ"] }, + "data_plane": "", + "directives": ["action"], + "diagnostics": [] + }, + { + "id": "core-url-userinfo-inert", + "name": "at sign in URL userinfo stays inert", + "source": "https://user@example.test/@model:atlas/atlas-4", + "options": {}, + "data_plane": "https://user@example.test/@model:atlas/atlas-4", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-escaped-action-inert", + "name": "escaped action sigil stays inert", + "source": "Use \\/translate and continue", + "options": { "actions": ["translate"] }, + "data_plane": "Use \\/translate and continue", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-windows-path-inert", + "name": "Windows and POSIX paths stay inert", + "source": "C:\\Users\\atlas\\notes and /usr/local/bin", + "options": {}, + "data_plane": "C:\\Users\\atlas\\notes and /usr/local/bin", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-uppercase-ps-tag-inert", + "name": "uppercase tag is outside the ps namespace", + "source": "text", + "options": {}, + "data_plane": "text", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-non-ps-quoted-angle-inert", + "name": "angle bracket inside a quoted foreign attribute does not end the tag", + "source": " @model:atlas/atlas-4\">text", + "options": {}, + "data_plane": " @model:atlas/atlas-4\">text", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-html-comment-angle-inert", + "name": "angle brackets inside an HTML comment stay inert", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-processing-instruction-angle-inert", + "name": "angle bracket inside a processing instruction stays inert", + "source": " @model:atlas/atlas-4\"?>", + "options": {}, + "data_plane": " @model:atlas/atlas-4\"?>", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-doctype-subset-inert", + "name": "directive-shaped text in a declaration subset stays inert", + "source": " @model:atlas/atlas-4]>", + "options": {}, + "data_plane": " @model:atlas/atlas-4]>", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-arabic-no-spacing-boundary", + "name": "Arabic prose without a space permits a point island", + "source": "ملخص@model:atlas/atlas-4", + "options": {}, + "data_plane": "ملخص", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-hebrew-no-spacing-boundary", + "name": "Hebrew prose without a space permits a point island", + "source": "סיכום@model:atlas/atlas-4", + "options": {}, + "data_plane": "סיכום", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-thai-no-spacing-boundary", + "name": "Thai prose without a space permits a point island", + "source": "สรุป@model:atlas/atlas-4", + "options": {}, + "data_plane": "สรุป", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-nfc-bare-action-resolution", + "name": "bare action resolution uses NFC", + "source": "/café", + "options": { "actions": ["café"] }, + "data_plane": "", + "directives": ["action"], + "diagnostics": [] + }, + { + "id": "core-bidi-lri-reference", + "name": "left-to-right isolate neutralizes a reference island", + "source": "@model:atlas/atlas-4\u2066x", + "options": {}, + "data_plane": "@model:atlas/atlas-4\u2066x", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-bidi-rli-action", + "name": "right-to-left isolate neutralizes an action island", + "source": "/trans\u2067late", + "options": { "actions": ["translate"] }, + "data_plane": "/trans\u2067late", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-bidi-fsi-reference", + "name": "first strong isolate neutralizes a reference island", + "source": "@file:notes\u2068.md", + "options": {}, + "data_plane": "@file:notes\u2068.md", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-bidi-pdf-action", + "name": "pop directional formatting neutralizes an action island", + "source": "/trans\u202clate", + "options": { "actions": ["translate"] }, + "data_plane": "/trans\u202clate", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-bidi-reference-argument", + "name": "bidi control in arguments neutralizes the whole reference island", + "source": "@tool:search(query: \"atlas\u2066\")", + "options": {}, + "data_plane": "@tool:search(query: \"atlas\u2066\")", + "directives": [], + "diagnostics": ["BIDI_CONTROL"] + }, + { + "id": "core-standalone-sigils-inert", + "name": "standalone and whitespace-separated sigils stay inert", + "source": "@ / ＠ ／ @ model / action", + "options": { "entities": ["model"], "actions": ["action"] }, + "data_plane": "@ / ＠ ／ @ model / action", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-double-colon-reference-invalid", + "name": "double colon reference fails closed", + "source": "@model::atlas", + "options": {}, + "data_plane": "@model::atlas", + "directives": [], + "diagnostics": ["SYNTAX_INVALID"] + }, + { + "id": "core-unclosed-reference-string", + "name": "unclosed reference string fails closed", + "source": "@tool:search(query: \"atlas)", + "options": {}, + "data_plane": "@tool:search(query: \"atlas)", + "directives": [], + "diagnostics": ["SYNTAX_INVALID"] + }, + { + "id": "core-unclosed-action-arguments", + "name": "unclosed action arguments fail closed", + "source": "/translate(to: \"km\"", + "options": { "actions": ["translate"] }, + "data_plane": "/translate(to: \"km\"", + "directives": [], + "diagnostics": ["SYNTAX_INVALID"] + }, + { + "id": "core-empty-action-arguments", + "name": "empty action argument list is accepted", + "source": "/translate()", + "options": { "actions": ["translate"] }, + "data_plane": "", + "directives": ["action"], + "diagnostics": [] + }, + { + "id": "core-signed-decimal-arguments", + "name": "signed and decimal argument values remain typed scalars", + "source": "@tool:search(offset: -2, ratio: 0.5)", + "options": {}, + "data_plane": "", + "directives": ["reference"], + "diagnostics": [] + }, + { + "id": "core-strict-fallback-route", + "name": "strict primary may have an explicit authored fallback", + "source": "@model:atlas/atlas-4! else @model:atlas/atlas-mini else fail", + "options": {}, + "data_plane": "", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-route-followed-by-punctuation", + "name": "punctuation after a route remains in the data plane", + "source": "(@model:atlas/atlas-4 else ask).", + "options": {}, + "data_plane": "().", + "directives": ["route"], + "diagnostics": [] + }, + { + "id": "core-empty-span", + "name": "empty ps span is a valid control island", + "source": "Before after", + "options": {}, + "data_plane": "Before after", + "directives": ["span"], + "diagnostics": [] + }, + { + "id": "core-span-multiple-references", + "name": "span header accepts multiple references", + "source": "Combine", + "options": {}, + "data_plane": "Combine", + "directives": ["span"], + "diagnostics": [] + }, + { + "id": "core-closing-span-tag-inert", + "name": "closing ps tag outside a span stays inert", + "source": "Before after", + "options": {}, + "data_plane": "Before after", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-self-closing-ps-span-invalid", + "name": "self-closing ps syntax fails closed as an unclosed span", + "source": "", + "options": {}, + "data_plane": "", + "directives": [], + "diagnostics": ["UNCLOSED_SPAN"] + }, + { + "id": "core-frontmatter-uppercase-marker-inert", + "name": "uppercase frontmatter marker stays content", + "source": "---PS\nversion: \"0.2\"\n---\nBody", + "options": {}, + "data_plane": "---PS\nversion: \"0.2\"\n---\nBody", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-frontmatter-leading-bom-inert", + "name": "byte order mark prevents document-start frontmatter activation", + "source": "---ps\nversion: \"0.2\"\n---\nBody", + "options": {}, + "data_plane": "---ps\nversion: \"0.2\"\n---\nBody", + "directives": [], + "diagnostics": [] + }, + { + "id": "core-frontmatter-nonclosing-marker", + "name": "closing marker with trailing text does not close frontmatter", + "source": "---ps\nversion: \"0.2\"\n--- trailing\nBody", + "options": {}, + "data_plane": "---ps\nversion: \"0.2\"\n--- trailing\nBody", + "directives": [], + "diagnostics": ["UNCLOSED_FRONTMATTER"] } ] diff --git a/conformance/cases/trace-producer.json b/conformance/cases/trace-producer.json new file mode 100644 index 0000000..c6b3902 --- /dev/null +++ b/conformance/cases/trace-producer.json @@ -0,0 +1,64 @@ +{ + "format_version": "0.1-draft", + "cases": [ + { + "id": "producer-kept", + "input": "conformance/fixtures/producer/input-kept.json", + "transcript": "conformance/fixtures/producer/transcript-kept.json", + "expected": { "outcome": "trace", "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-fallback", + "input": "conformance/fixtures/producer/input-fallback.json", + "transcript": "conformance/fixtures/producer/transcript-fallback.json", + "expected": { "outcome": "trace", "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-substituted", + "input": "conformance/fixtures/producer/input-substituted.json", + "transcript": "conformance/fixtures/producer/transcript-substituted.json", + "expected": { "outcome": "trace", "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-refused", + "input": "conformance/fixtures/producer/input-refused.json", + "transcript": "conformance/fixtures/producer/transcript-refused.json", + "expected": { "outcome": "trace", "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-boundary-failed", + "input": "conformance/fixtures/producer/input-boundary-failed.json", + "transcript": "conformance/fixtures/producer/transcript-boundary-failed.json", + "expected": { "outcome": "trace", "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-boundary-succeeded", + "input": "conformance/fixtures/producer/input-boundary-succeeded.json", + "transcript": "conformance/fixtures/producer/transcript-boundary-succeeded.json", + "expected": { "outcome": "trace", "conformant": true, "diagnostics": [] } + }, + { + "id": "producer-strict-substitution-rejected", + "input": "conformance/fixtures/producer/input-strict-substitution.json", + "expected": { + "outcome": "error", + "code": "TRACE_PRODUCER_SUBSTITUTION_NOT_AUTHORIZED" + } + }, + { + "id": "producer-multiple-filled-rejected", + "input": "conformance/fixtures/producer/input-multiple-filled.json", + "expected": { "outcome": "error", "code": "TRACE_PRODUCER_MULTIPLE_FILLED" } + }, + { + "id": "producer-missing-refusal-rejected", + "input": "conformance/fixtures/producer/input-missing-refusal.json", + "expected": { "outcome": "error", "code": "TRACE_PRODUCER_REFUSAL_REQUIRED" } + }, + { + "id": "producer-refusal-conflict-rejected", + "input": "conformance/fixtures/producer/input-refusal-conflict.json", + "expected": { "outcome": "error", "code": "TRACE_PRODUCER_REFUSAL_CONFLICT" } + } + ] +} diff --git a/conformance/core-adapter-result.schema.json b/conformance/core-adapter-result.schema.json new file mode 100644 index 0000000..6f2f490 --- /dev/null +++ b/conformance/core-adapter-result.schema.json @@ -0,0 +1,289 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/core-adapter-result/0.1-draft", + "title": "PromptSyntax normalized Core parser adapter result", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "target", "implementation", "results"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "target": { "const": "core-parser" }, + "implementation": { "$ref": "#/$defs/implementation" }, + "results": { + "type": "array", + "items": { "$ref": "#/$defs/case_result" } + } + }, + "$defs": { + "implementation": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "commit"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" } + } + }, + "case_result": { + "type": "object", + "additionalProperties": false, + "required": ["case_id", "conformant", "diagnostics", "output"], + "properties": { + "case_id": { "type": "string", "pattern": "^core-[a-z0-9]+(?:-[a-z0-9]+)*$" }, + "conformant": { "type": "boolean" }, + "diagnostics": { + "type": "array", + "items": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" } + }, + "output": { "$ref": "#/$defs/output" } + } + }, + "output": { + "type": "object", + "additionalProperties": false, + "required": [ + "round_trip", + "data_plane", + "segments", + "directives", + "parser_diagnostics" + ], + "properties": { + "round_trip": { "type": "string" }, + "data_plane": { "type": "string" }, + "segments": { "type": "array", "items": { "$ref": "#/$defs/segment" } }, + "directives": { + "type": "array", + "items": { "$ref": "#/$defs/directive_record" } + }, + "parser_diagnostics": { + "type": "array", + "items": { "$ref": "#/$defs/parser_diagnostic" } + } + } + }, + "span": { + "type": "object", + "additionalProperties": false, + "required": ["start", "end", "source"], + "properties": { + "start": { "type": "integer", "minimum": 0 }, + "end": { "type": "integer", "minimum": 0 }, + "source": { "type": "string" } + } + }, + "segment": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["type", "span", "source"], + "properties": { + "type": { "const": "text" }, + "span": { "$ref": "#/$defs/span" }, + "source": { "type": "string" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "span", "source", "directive"], + "properties": { + "type": { "const": "directive" }, + "span": { "$ref": "#/$defs/span" }, + "source": { "type": "string" }, + "directive": { "$ref": "#/$defs/directive" } + } + } + ] + }, + "directive_record": { + "type": "object", + "additionalProperties": false, + "required": ["span", "source", "directive"], + "properties": { + "span": { "$ref": "#/$defs/span" }, + "source": { "type": "string" }, + "directive": { "$ref": "#/$defs/directive" } + } + }, + "parser_diagnostic": { + "type": "object", + "additionalProperties": false, + "required": ["code", "span"], + "properties": { + "code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" }, + "span": { "$ref": "#/$defs/span" } + } + }, + "directive": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "reference"], + "properties": { + "kind": { "const": "reference" }, + "reference": { "$ref": "#/$defs/reference" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "action"], + "properties": { + "kind": { "const": "action" }, + "action": { "$ref": "#/$defs/action" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "route"], + "properties": { + "kind": { "const": "route" }, + "route": { "$ref": "#/$defs/route" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "header", "segments"], + "properties": { + "kind": { "const": "span" }, + "header": { "$ref": "#/$defs/span_header" }, + "segments": { "type": "array", "items": { "$ref": "#/$defs/segment" } } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "reference"], + "properties": { + "kind": { "const": "authoring_segment" }, + "reference": { "$ref": "#/$defs/reference" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "header"], + "properties": { + "kind": { "const": "invalid_authoring_segment" }, + "header": { "type": "string" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "body"], + "properties": { + "kind": { "const": "frontmatter" }, + "body": { "type": "string" } + } + } + ] + }, + "reference": { + "type": "object", + "additionalProperties": false, + "required": ["namespace", "name", "version", "strict", "arguments"], + "properties": { + "namespace": { "type": ["string", "null"] }, + "name": { "type": "string", "minLength": 1 }, + "version": { "type": ["string", "null"] }, + "strict": { "type": "boolean" }, + "arguments": { "type": "array", "items": { "$ref": "#/$defs/argument" } } + } + }, + "action": { + "type": "object", + "additionalProperties": false, + "required": ["name", "arguments"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "arguments": { "type": "array", "items": { "$ref": "#/$defs/argument" } } + } + }, + "route": { + "type": "object", + "additionalProperties": false, + "required": ["steps", "terminal"], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["reference", "limits"], + "properties": { + "reference": { "$ref": "#/$defs/reference" }, + "limits": { "type": "array", "items": { "$ref": "#/$defs/argument" } } + } + } + }, + "terminal": { "enum": ["ask", "fail", null] } + } + }, + "span_header": { + "type": "object", + "additionalProperties": false, + "required": ["raw", "references", "attributes"], + "properties": { + "raw": { "type": "string" }, + "references": { "type": "array", "items": { "$ref": "#/$defs/reference" } }, + "attributes": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["key", "value"], + "properties": { + "key": { "type": "string", "minLength": 1 }, + "value": { "$ref": "#/$defs/scalar" } + } + } + } + } + }, + "argument": { + "type": "object", + "additionalProperties": false, + "required": ["key", "value"], + "properties": { + "key": { "type": "string", "minLength": 1 }, + "value": { "$ref": "#/$defs/scalar" } + } + }, + "scalar": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "value"], + "properties": { + "kind": { "enum": ["string", "number", "bare"] }, + "value": { "type": "string" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "value"], + "properties": { + "kind": { "const": "boolean" }, + "value": { "type": "boolean" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { "kind": { "const": "null" } } + } + ] + } + } +} diff --git a/conformance/core-cases.schema.json b/conformance/core-cases.schema.json index cf170d2..8ab32b5 100644 --- a/conformance/core-cases.schema.json +++ b/conformance/core-cases.schema.json @@ -7,8 +7,9 @@ "items": { "type": "object", "additionalProperties": false, - "required": ["name", "source", "options", "data_plane", "directives", "diagnostics"], + "required": ["id", "name", "source", "options", "data_plane", "directives", "diagnostics"], "properties": { + "id": { "type": "string", "pattern": "^core-[a-z0-9]+(?:-[a-z0-9]+)*$" }, "name": { "type": "string", "minLength": 1 }, "source": { "type": "string" }, "options": { diff --git a/conformance/core-differential-input.schema.json b/conformance/core-differential-input.schema.json new file mode 100644 index 0000000..18d5ae4 --- /dev/null +++ b/conformance/core-differential-input.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/core-differential-input/0.1-draft", + "title": "PromptSyntax generated Core differential inputs", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "source", "options"], + "properties": { + "id": { "type": "string", "pattern": "^core-generated-[0-9]{6}$" }, + "name": { "type": "string", "minLength": 1 }, + "source": { "type": "string" }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "entities": { "$ref": "#/$defs/string_set" }, + "actions": { "$ref": "#/$defs/string_set" }, + "authoring_namespaces": { "$ref": "#/$defs/string_set" } + } + } + } + }, + "$defs": { + "string_set": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string" } + } + } +} diff --git a/conformance/core-differential-report.schema.json b/conformance/core-differential-report.schema.json new file mode 100644 index 0000000..2afd1e6 --- /dev/null +++ b/conformance/core-differential-report.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/core-differential-report/0.1-draft", + "title": "PromptSyntax Core parser differential report", + "type": "object", + "additionalProperties": false, + "required": [ + "format_version", + "target", + "generator", + "compatible", + "left", + "right", + "case_count", + "matched_case_count", + "diagnostics" + ], + "properties": { + "format_version": { "const": "0.1-draft" }, + "target": { "const": "core-parser-differential" }, + "generator": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "seed"], + "properties": { + "id": { "const": "promptsyntax-core-grammar" }, + "version": { "const": "0.1-draft" }, + "seed": { "type": "string", "pattern": "^[0-9]+$" } + } + }, + { "type": "null" } + ] + }, + "compatible": { "type": "boolean" }, + "left": { "oneOf": [{ "$ref": "#/$defs/implementation" }, { "type": "null" }] }, + "right": { "oneOf": [{ "$ref": "#/$defs/implementation" }, { "type": "null" }] }, + "case_count": { "type": "integer", "minimum": 0 }, + "matched_case_count": { "type": "integer", "minimum": 0 }, + "diagnostics": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "pointer", "message"], + "properties": { + "code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" }, + "pointer": { "type": "string" }, + "message": { "type": "string" } + } + } + } + }, + "$defs": { + "implementation": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "commit"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" } + } + } + } +} diff --git a/conformance/diagnostic-registry.json b/conformance/diagnostic-registry.json index c7013e8..389e16f 100644 --- a/conformance/diagnostic-registry.json +++ b/conformance/diagnostic-registry.json @@ -103,6 +103,150 @@ "severity": "error", "description": "A requirement lists the same conformance target more than once." }, + { + "code": "CORE_ADAPTER_FORMAT_UNSUPPORTED", + "phase": "adapter", + "severity": "error", + "description": "A Core parser adapter uses an unsupported result format version." + }, + { + "code": "CORE_ADAPTER_IMPLEMENTATION_INVALID", + "phase": "adapter", + "severity": "error", + "description": "A Core parser adapter supplies invalid implementation identity, version, or commit metadata." + }, + { + "code": "CORE_ADAPTER_IMPLEMENTATIONS_IDENTICAL", + "phase": "adapter", + "severity": "error", + "description": "A differential comparison was given the same implementation identity, version, and commit on both sides." + }, + { + "code": "CORE_ADAPTER_TARGET_INVALID", + "phase": "adapter", + "severity": "error", + "description": "A Core parser differential input does not declare the core-parser target." + }, + { + "code": "CORE_CASE_ID_DUPLICATE", + "phase": "adapter", + "severity": "error", + "description": "A Core parser adapter emits the same canonical case identifier more than once." + }, + { + "code": "CORE_CASES_ID_DUPLICATE", + "phase": "corpus", + "severity": "error", + "description": "The canonical Core parser corpus contains a duplicate case identifier." + }, + { + "code": "CORE_CASES_JSON_INVALID", + "phase": "corpus", + "severity": "error", + "description": "The canonical Core parser corpus cannot be read as a case array." + }, + { + "code": "CORE_CASES_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "The canonical Core parser corpus could not be read." + }, + { + "code": "CORE_CASE_MISSING_LEFT", + "phase": "adapter", + "severity": "error", + "description": "A Core parser case emitted by the right adapter is absent from the left adapter output." + }, + { + "code": "CORE_CASE_MISSING_RIGHT", + "phase": "adapter", + "severity": "error", + "description": "A Core parser case emitted by the left adapter is absent from the right adapter output." + }, + { + "code": "CORE_CASE_NONCONFORMANT_LEFT", + "phase": "adapter", + "severity": "error", + "description": "The left Core parser adapter disagrees with a specification-owned expected outcome." + }, + { + "code": "CORE_CASE_NONCONFORMANT_RIGHT", + "phase": "adapter", + "severity": "error", + "description": "The right Core parser adapter disagrees with a specification-owned expected outcome." + }, + { + "code": "CORE_CASE_UNEXPECTED_LEFT", + "phase": "adapter", + "severity": "error", + "description": "The left Core parser adapter emits a case absent from the canonical corpus." + }, + { + "code": "CORE_CASE_UNEXPECTED_RIGHT", + "phase": "adapter", + "severity": "error", + "description": "The right Core parser adapter emits a case absent from the canonical corpus." + }, + { + "code": "CORE_CASE_EXPECTATION_INCOMPLETE", + "phase": "adapter", + "severity": "error", + "description": "A Core parser case supplies only part of the canonical expected-answer tuple." + }, + { + "code": "CORE_DATA_PLANE_MISMATCH", + "phase": "adapter", + "severity": "error", + "description": "A Core parser data-plane projection disagrees with the canonical case." + }, + { + "code": "CORE_DIAGNOSTIC_SEQUENCE_MISMATCH", + "phase": "adapter", + "severity": "error", + "description": "A Core parser diagnostic sequence disagrees with the canonical case." + }, + { + "code": "CORE_DIRECTIVE_SEQUENCE_MISMATCH", + "phase": "adapter", + "severity": "error", + "description": "A Core parser directive sequence disagrees with the canonical case." + }, + { + "code": "CORE_OUTPUT_DIFFERENTIAL", + "phase": "adapter", + "severity": "error", + "description": "Two Core parser adapters produce different normalized outputs for the same canonical case." + }, + { + "code": "CORE_ROUND_TRIP_MISMATCH", + "phase": "adapter", + "severity": "error", + "description": "A Core parser does not reproduce the canonical authored source exactly." + }, + { + "code": "LEFT_ADAPTER_JSON_INVALID", + "phase": "adapter", + "severity": "error", + "description": "The left Core parser adapter result is not valid closed-model JSON." + }, + { + "code": "LEFT_ADAPTER_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "The left Core parser adapter result could not be read." + }, + { + "code": "RIGHT_ADAPTER_JSON_INVALID", + "phase": "adapter", + "severity": "error", + "description": "The right Core parser adapter result is not valid closed-model JSON." + }, + { + "code": "RIGHT_ADAPTER_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "The right Core parser adapter result could not be read." + }, { "code": "INSTANCE_JSON_INVALID", "phase": "instance", @@ -313,6 +457,102 @@ "severity": "error", "description": "Inline content exceeds the trace's declared UTF-8 byte threshold." }, + { + "code": "TRACE_PRODUCER_ATTEMPT_ORDER_INVALID", + "phase": "producer", + "severity": "error", + "description": "A deterministic routing transcript contains an attempt after its filled attempt." + }, + { + "code": "TRACE_PRODUCER_ATTEMPT_ROUTE_MISMATCH", + "phase": "producer", + "severity": "error", + "description": "A deterministic routing attempt does not align with the corresponding authored route step." + }, + { + "code": "TRACE_PRODUCER_EVENT_ID_DUPLICATE", + "phase": "producer", + "severity": "error", + "description": "Trace producer input repeats an inference or boundary event identifier." + }, + { + "code": "TRACE_PRODUCER_INPUT_INVALID", + "phase": "producer", + "severity": "error", + "description": "Trace producer input is malformed or lacks a required deterministic fact." + }, + { + "code": "TRACE_PRODUCER_INPUT_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "A Trace producer input fixture could not be read." + }, + { + "code": "TRACE_PRODUCER_INPUT_UNSUPPORTED", + "phase": "producer", + "severity": "error", + "description": "Trace producer input requests a tier, mode, or profile outside the producer contract." + }, + { + "code": "TRACE_PRODUCER_MULTIPLE_FILLED", + "phase": "producer", + "severity": "error", + "description": "Deterministic routing facts report more than one filled attempt for one inference." + }, + { + "code": "TRACE_PRODUCER_OUTPUT_INVALID", + "phase": "producer", + "severity": "error", + "description": "A Trace producer process did not emit its required JSON result envelope." + }, + { + "code": "TRACE_PRODUCER_PROCESS_FAILED", + "phase": "producer", + "severity": "error", + "description": "A Trace producer process could not be started or exited contrary to its expected outcome." + }, + { + "code": "TRACE_PRODUCER_REFUSAL_CONFLICT", + "phase": "producer", + "severity": "error", + "description": "Deterministic routing facts contain both a filled attempt and a refusal fact." + }, + { + "code": "TRACE_PRODUCER_REFUSAL_REQUIRED", + "phase": "producer", + "severity": "error", + "description": "Deterministic routing facts with no filled attempt omit the required refusal fact." + }, + { + "code": "TRACE_PRODUCER_ROUTE_DUPLICATE", + "phase": "producer", + "severity": "error", + "description": "A Trace producer input repeats a canonical entity in one authored route." + }, + { + "code": "TRACE_PRODUCER_ROUTE_INVALID", + "phase": "producer", + "severity": "error", + "description": "A Trace producer input lacks a route step or execution attempt." + }, + { + "code": "TRACE_PRODUCER_SUBSTITUTION_NOT_AUTHORIZED", + "phase": "producer", + "severity": "error", + "description": "Execution facts report an entity outside the authored route without explicit best-effort authorization." + }, + { + "code": "TRACE_PRODUCER_TRANSCRIPT_READ_FAILED", + "phase": "io", + "severity": "error", + "description": "An independent Trace producer transcript fixture could not be read." + }, + { + "code": "TRACE_PRODUCER_TRANSCRIPT_REQUIRED", + "phase": "producer", + "severity": "error", + "description": "A successful Trace producer adapter case lacks an independent transcript." + }, { "code": "TRACE_PRODUCER_ARTIFACT_DUPLICATE", "phase": "producer", diff --git a/conformance/diagnostic-registry.schema.json b/conformance/diagnostic-registry.schema.json index 9fe53b0..c30b58b 100644 --- a/conformance/diagnostic-registry.schema.json +++ b/conformance/diagnostic-registry.schema.json @@ -16,7 +16,9 @@ "required": ["code", "phase", "severity", "description"], "properties": { "code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" }, - "phase": { "enum": ["io", "corpus", "schema", "instance", "semantic", "producer"] }, + "phase": { + "enum": ["io", "corpus", "schema", "instance", "semantic", "producer", "adapter"] + }, "severity": { "enum": ["error", "warning"] }, "description": { "type": "string", "minLength": 1 } } diff --git a/conformance/fixtures/producer/input-boundary-failed.json b/conformance/fixtures/producer/input-boundary-failed.json new file mode 100644 index 0000000..c172cb3 --- /dev/null +++ b/conformance/fixtures/producer/input-boundary-failed.json @@ -0,0 +1,63 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-boundary-failed-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-boundary-failed-001", + "kind": "user-initiated", + "compiled_request_utf8": "Find the latest customer note.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.017, "wall_time_ms": 2010 } + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [ + { + "id": "boundary-failed-001", + "tool": "tool:example/search@1", + "outcome": "failed", + "sent": { "state": "inline", "text": "latest customer note" }, + "reason": "ENTITY_UNAVAILABLE", + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "status": "refused", + "reason": "ENTITY_UNAVAILABLE", + "deciding_authority": "venue-operations" + } + ], + "measured": { "wall_time_ms": 5 } + } + ] +} diff --git a/conformance/fixtures/producer/input-boundary-succeeded.json b/conformance/fixtures/producer/input-boundary-succeeded.json new file mode 100644 index 0000000..a0637d6 --- /dev/null +++ b/conformance/fixtures/producer/input-boundary-succeeded.json @@ -0,0 +1,63 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-boundary-succeeded-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-boundary-succeeded-001", + "kind": "user-initiated", + "compiled_request_utf8": "Find the latest customer note.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.017, "wall_time_ms": 2010 } + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [ + { + "id": "boundary-succeeded-001", + "tool": "tool:example/search@1", + "outcome": "succeeded", + "sent": { "state": "inline", "text": "latest customer note" }, + "received": { "state": "inline", "text": "Customer requested export controls." }, + "fill": [ + { + "kind": "action", + "requested": "tool:example/search@1", + "applied": "tool:example/search@1", + "status": "applied" + } + ], + "internal": "outside-trace-perimeter", + "measured": { "wall_time_ms": 18 } + } + ] +} diff --git a/conformance/fixtures/producer/input-fallback.json b/conformance/fixtures/producer/input-fallback.json new file mode 100644 index 0000000..5020a9c --- /dev/null +++ b/conformance/fixtures/producer/input-fallback.json @@ -0,0 +1,56 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-fallback-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-fallback-001", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize the customer notes.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01! else @model:example/atlas-mini@2026-08-01", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + }, + { + "ref": "@model:example/atlas-mini@2026-08-01", + "canonical": "model:example/atlas-mini@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "outcome": "failed", + "reasons": ["ENTITY_UNAVAILABLE"] + }, + { + "ref": "@model:example/atlas-mini@2026-08-01", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.009, "wall_time_ms": 1410 } + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-kept.json b/conformance/fixtures/producer/input-kept.json new file mode 100644 index 0000000..36bfe4a --- /dev/null +++ b/conformance/fixtures/producer/input-kept.json @@ -0,0 +1,56 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-kept-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "vendor_profile": "vendor-profile/example-2026-08" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "profile": "trace-user-0-3-draft" + }, + "inferences": [ + { + "id": "inference-kept-001", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize the report.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.041, "wall_time_ms": 3210 } + } + ], + "non_entity_fill": [ + { + "kind": "parameter", + "name": "temperature", + "requested": 0.2, + "applied": 0.2, + "status": "applied", + "policy": "strict" + } + ] + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-missing-refusal.json b/conformance/fixtures/producer/input-missing-refusal.json new file mode 100644 index 0000000..d3cb7b4 --- /dev/null +++ b/conformance/fixtures/producer/input-missing-refusal.json @@ -0,0 +1,41 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-invalid-missing-refusal", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { "id": "venue:example", "capability_document": "https://venue.example/cap.json" }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-invalid-missing-refusal", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "outcome": "failed", + "reasons": ["ENTITY_UNAVAILABLE"] + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-multiple-filled.json b/conformance/fixtures/producer/input-multiple-filled.json new file mode 100644 index 0000000..bcbea16 --- /dev/null +++ b/conformance/fixtures/producer/input-multiple-filled.json @@ -0,0 +1,54 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-invalid-multiple-filled", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { "id": "venue:example", "capability_document": "https://venue.example/cap.json" }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-invalid-multiple-filled", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01 else @model:example/atlas-mini@2026-08-01", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + }, + { + "ref": "@model:example/atlas-mini@2026-08-01", + "canonical": "model:example/atlas-mini@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { "wall_time_ms": 10 } + }, + { + "ref": "@model:example/atlas-mini@2026-08-01", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "wall_time_ms": 7 } + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-refusal-conflict.json b/conformance/fixtures/producer/input-refusal-conflict.json new file mode 100644 index 0000000..af36a37 --- /dev/null +++ b/conformance/fixtures/producer/input-refusal-conflict.json @@ -0,0 +1,47 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-invalid-refusal-conflict", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { "id": "venue:example", "capability_document": "https://venue.example/cap.json" }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-invalid-refusal-conflict", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-4@2026-08-01", + "outcome": "filled", + "measured": { "wall_time_ms": 10 } + } + ], + "non_entity_fill": [], + "refusal": { + "reason": "FALLBACK_EXHAUSTED", + "authority": "venue-operations", + "recourse": { "available": true, "kind": "retry" } + } + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-refused.json b/conformance/fixtures/producer/input-refused.json new file mode 100644 index 0000000..b3c65f3 --- /dev/null +++ b/conformance/fixtures/producer/input-refused.json @@ -0,0 +1,49 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-refused-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-refused-001", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize the restricted report.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "outcome": "failed", + "reasons": ["ENTITY_UNAVAILABLE"] + } + ], + "non_entity_fill": [], + "refusal": { + "reason": "FALLBACK_EXHAUSTED", + "authority": "venue-operations", + "recourse": { "available": true, "kind": "retry" } + } + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-strict-substitution.json b/conformance/fixtures/producer/input-strict-substitution.json new file mode 100644 index 0000000..6e38879 --- /dev/null +++ b/conformance/fixtures/producer/input-strict-substitution.json @@ -0,0 +1,42 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-invalid-strict-substitution", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { "id": "venue:example", "capability_document": "https://venue.example/cap.json" }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-invalid-strict-substitution", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01!", + "policy": "strict", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01!", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "wall_time_ms": 10 } + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/input-substituted.json b/conformance/fixtures/producer/input-substituted.json new file mode 100644 index 0000000..da3ccbd --- /dev/null +++ b/conformance/fixtures/producer/input-substituted.json @@ -0,0 +1,45 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-substituted-001", + "tier": "user", + "mode": "executed", + "content_mode": "tiered", + "venue": { + "id": "venue:example", + "capability_document": "https://venue.example/ps-capabilities/0.3.json" + }, + "coverage": { + "routing": "complete", + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096 + }, + "inferences": [ + { + "id": "inference-substituted-001", + "kind": "user-initiated", + "compiled_request_utf8": "Summarize the risk report.", + "routing": { + "requested": "@model:example/atlas-4@2026-08-01", + "policy": "best-effort", + "route": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "canonical": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ], + "attempts": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-mini@2026-08-01", + "outcome": "filled", + "measured": { "cost_usd": 0.008, "wall_time_ms": 1330 } + } + ], + "non_entity_fill": [] + } + } + ], + "boundaries": [] +} diff --git a/conformance/fixtures/producer/transcript-boundary-failed.json b/conformance/fixtures/producer/transcript-boundary-failed.json new file mode 100644 index 0000000..6d3b255 --- /dev/null +++ b/conformance/fixtures/producer/transcript-boundary-failed.json @@ -0,0 +1,16 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-boundary-failed-001", + "inferences": [ + { + "id": "inference-boundary-failed-001", + "actual_request_utf8": "Find the latest customer note.", + "routing": { + "outcome": "invoked", + "bound": "model:example/atlas-4@2026-08-01" + } + } + ], + "boundaries": [{ "id": "boundary-failed-001", "outcome": "failed" }], + "external_artifacts": [] +} diff --git a/conformance/fixtures/producer/transcript-boundary-succeeded.json b/conformance/fixtures/producer/transcript-boundary-succeeded.json new file mode 100644 index 0000000..4df3470 --- /dev/null +++ b/conformance/fixtures/producer/transcript-boundary-succeeded.json @@ -0,0 +1,16 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-boundary-succeeded-001", + "inferences": [ + { + "id": "inference-boundary-succeeded-001", + "actual_request_utf8": "Find the latest customer note.", + "routing": { + "outcome": "invoked", + "bound": "model:example/atlas-4@2026-08-01" + } + } + ], + "boundaries": [{ "id": "boundary-succeeded-001", "outcome": "succeeded" }], + "external_artifacts": [] +} diff --git a/conformance/fixtures/producer/transcript-fallback.json b/conformance/fixtures/producer/transcript-fallback.json new file mode 100644 index 0000000..f871421 --- /dev/null +++ b/conformance/fixtures/producer/transcript-fallback.json @@ -0,0 +1,16 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-fallback-001", + "inferences": [ + { + "id": "inference-fallback-001", + "actual_request_utf8": "Summarize the customer notes.", + "routing": { + "outcome": "invoked", + "bound": "model:example/atlas-mini@2026-08-01" + } + } + ], + "boundaries": [], + "external_artifacts": [] +} diff --git a/conformance/fixtures/producer/transcript-kept.json b/conformance/fixtures/producer/transcript-kept.json new file mode 100644 index 0000000..ca39927 --- /dev/null +++ b/conformance/fixtures/producer/transcript-kept.json @@ -0,0 +1,16 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-kept-001", + "inferences": [ + { + "id": "inference-kept-001", + "actual_request_utf8": "Summarize the report.", + "routing": { + "outcome": "invoked", + "bound": "model:example/atlas-4@2026-08-01" + } + } + ], + "boundaries": [], + "external_artifacts": [] +} diff --git a/conformance/fixtures/producer/transcript-refused.json b/conformance/fixtures/producer/transcript-refused.json new file mode 100644 index 0000000..6b07128 --- /dev/null +++ b/conformance/fixtures/producer/transcript-refused.json @@ -0,0 +1,13 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-refused-001", + "inferences": [ + { + "id": "inference-refused-001", + "actual_request_utf8": "Summarize the restricted report.", + "routing": { "outcome": "refused" } + } + ], + "boundaries": [], + "external_artifacts": [] +} diff --git a/conformance/fixtures/producer/transcript-substituted.json b/conformance/fixtures/producer/transcript-substituted.json new file mode 100644 index 0000000..5cd9598 --- /dev/null +++ b/conformance/fixtures/producer/transcript-substituted.json @@ -0,0 +1,16 @@ +{ + "format_version": "0.1-draft", + "turn": "turn-producer-substituted-001", + "inferences": [ + { + "id": "inference-substituted-001", + "actual_request_utf8": "Summarize the risk report.", + "routing": { + "outcome": "invoked", + "bound": "model:example/atlas-mini@2026-08-01" + } + } + ], + "boundaries": [], + "external_artifacts": [] +} diff --git a/conformance/implementations.lock.json b/conformance/implementations.lock.json index bd7c92d..e5684a5 100644 --- a/conformance/implementations.lock.json +++ b/conformance/implementations.lock.json @@ -4,20 +4,37 @@ { "id": "promptsyntax-rs", "repository": "pathscale/PromptSyntax-rs", - "commit": "c64072f618ffdb53cc389fc40dfe0370c4cf05be", - "adapter": "tests/conformance.rs", + "commit": "2789fac8acb810279ece887f46c025ef987ce0bf", + "adapter": "src/bin/ps-core-adapter.rs", "fixture": "tests/conformance.json", "runtime": { "name": "rust", "version": "1.97.0" }, - "command": ["cargo", "test"] + "command": [ + "cargo", "run", "--quiet", "--release", "--locked", "--bin", + "ps-core-adapter", "--", "{cases}", "{commit}" + ], + "stream_command": [ + "cargo", "run", "--quiet", "--release", "--locked", "--bin", + "ps-core-adapter", "--", "{cases}", "{commit}", "--jsonl" + ], + "trace_adapter": "src/bin/ps-trace-producer.rs", + "trace_command": [ + "cargo", "run", "--quiet", "--release", "--locked", "--bin", + "ps-trace-producer", "--", "{input}" + ] }, { "id": "promptsyntax-ts", "repository": "pathscale/PromptSyntax-ts", - "commit": "b22acfac066c0ea6eec7d042d69040c695ef5422", - "adapter": "test/parser.test.ts", + "commit": "3c2c505d6f8fa836aa42abe12c0b466bcc3ba254", + "adapter": "src/bin/core-adapter.ts", "fixture": "test/conformance.json", "runtime": { "name": "bun", "version": "1.3.14" }, - "command": ["bun", "test"] + "command": ["bun", "run", "core-adapter", "{cases}", "{commit}"], + "stream_command": [ + "bun", "run", "core-adapter", "{cases}", "{commit}", "--jsonl" + ], + "trace_adapter": "src/bin/trace-producer.ts", + "trace_command": ["bun", "run", "trace-producer", "{input}"] } ] } diff --git a/conformance/implementations.schema.json b/conformance/implementations.schema.json index e7690ec..54f792d 100644 --- a/conformance/implementations.schema.json +++ b/conformance/implementations.schema.json @@ -13,7 +13,18 @@ "items": { "type": "object", "additionalProperties": false, - "required": ["id", "repository", "commit", "adapter", "fixture", "runtime", "command"], + "required": [ + "id", + "repository", + "commit", + "adapter", + "fixture", + "runtime", + "command", + "stream_command", + "trace_adapter", + "trace_command" + ], "properties": { "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" }, "repository": { "type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" }, @@ -33,6 +44,17 @@ "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } + }, + "stream_command": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "trace_adapter": { "type": "string", "minLength": 1 }, + "trace_command": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } } } } diff --git a/conformance/manifest.json b/conformance/manifest.json index 6171d0f..433c3b6 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -19,15 +19,25 @@ "trace_schema": "schemas/prompt-trace-0.3-draft.schema.json", "family_schema": "conformance/case-family.schema.json", "transcript_schema": "conformance/transcript.schema.json", + "trace_producer_input_schema": "conformance/trace-producer-input.schema.json", + "trace_producer_cases": "conformance/cases/trace-producer.json", + "trace_producer_cases_schema": "conformance/trace-producer-cases.schema.json", "profile": "conformance/profiles/trace-user-0.3-draft.json", "profile_schema": "conformance/profile.schema.json", "report_schema": "conformance/report.schema.json", "core_cases": "conformance/cases/core-parser.json", "core_cases_schema": "conformance/core-cases.schema.json", + "core_differential_input_schema": "conformance/core-differential-input.schema.json", + "core_adapter_result_schema": "conformance/core-adapter-result.schema.json", + "core_differential_report_schema": "conformance/core-differential-report.schema.json", "adapter_result_schema": "conformance/adapter-result.schema.json", "implementations_lock": "conformance/implementations.lock.json", "implementations_schema": "conformance/implementations.schema.json", - "candidate_report": "conformance/reports/0.1.0-rc.1.json" + "candidate_report": "conformance/reports/0.1.0-rc.1.json", + "core_differential_report": "conformance/reports/core-differential-0.1.0-rc.1.json", + "core_generated_differential_report": "conformance/reports/core-generated-differential-0.1.0-rc.1.json", + "trace_producer_rs_report": "conformance/reports/trace-producer-rs-0.1.0-rc.1.json", + "trace_producer_ts_report": "conformance/reports/trace-producer-ts-0.1.0-rc.1.json" }, "families": [ { diff --git a/conformance/manifest.schema.json b/conformance/manifest.schema.json index ccc1859..92b28b1 100644 --- a/conformance/manifest.schema.json +++ b/conformance/manifest.schema.json @@ -39,15 +39,25 @@ "trace_schema", "family_schema", "transcript_schema", + "trace_producer_input_schema", + "trace_producer_cases", + "trace_producer_cases_schema", "profile", "profile_schema", "report_schema", "core_cases", "core_cases_schema", + "core_differential_input_schema", + "core_adapter_result_schema", + "core_differential_report_schema", "adapter_result_schema", "implementations_lock", "implementations_schema", - "candidate_report" + "candidate_report", + "core_differential_report", + "core_generated_differential_report", + "trace_producer_rs_report", + "trace_producer_ts_report" ], "properties": { "requirements": { "type": "string", "minLength": 1 }, @@ -57,15 +67,25 @@ "trace_schema": { "type": "string", "minLength": 1 }, "family_schema": { "type": "string", "minLength": 1 }, "transcript_schema": { "type": "string", "minLength": 1 }, + "trace_producer_input_schema": { "type": "string", "minLength": 1 }, + "trace_producer_cases": { "type": "string", "minLength": 1 }, + "trace_producer_cases_schema": { "type": "string", "minLength": 1 }, "profile": { "type": "string", "minLength": 1 }, "profile_schema": { "type": "string", "minLength": 1 }, "report_schema": { "type": "string", "minLength": 1 }, "core_cases": { "type": "string", "minLength": 1 }, "core_cases_schema": { "type": "string", "minLength": 1 }, + "core_differential_input_schema": { "type": "string", "minLength": 1 }, + "core_adapter_result_schema": { "type": "string", "minLength": 1 }, + "core_differential_report_schema": { "type": "string", "minLength": 1 }, "adapter_result_schema": { "type": "string", "minLength": 1 }, "implementations_lock": { "type": "string", "minLength": 1 }, "implementations_schema": { "type": "string", "minLength": 1 }, - "candidate_report": { "type": "string", "minLength": 1 } + "candidate_report": { "type": "string", "minLength": 1 }, + "core_differential_report": { "type": "string", "minLength": 1 }, + "core_generated_differential_report": { "type": "string", "minLength": 1 }, + "trace_producer_rs_report": { "type": "string", "minLength": 1 }, + "trace_producer_ts_report": { "type": "string", "minLength": 1 } } }, "families": { diff --git a/conformance/releases/0.1.0-rc.1.md b/conformance/releases/0.1.0-rc.1.md index efce841..1f2421c 100644 --- a/conformance/releases/0.1.0-rc.1.md +++ b/conformance/releases/0.1.0-rc.1.md @@ -10,7 +10,13 @@ been issued. - Complete coverage of the 22 requirements in `trace-user-0-3-draft`. - Independent producer transcripts for UTF-8 request bytes, invoked entities, boundary outcomes, and external artifact bytes. -- One canonical Core parser corpus exercised by pinned Rust and TypeScript implementations. +- 101 canonical Core parser cases exercised by pinned Rust and TypeScript implementations. +- Full normalized-output differential comparison across segments, AST nodes, UTF-8 byte + spans, source slices, projections, and diagnostics. +- A deterministic 100,000-input grammar-generated differential run across 32 syntax + families, with zero expected answers embedded in generated inputs. +- Ten real-producer adapter cases, including six independently replayed valid traces and + four required typed rejections. - A deterministic JSON report with no wall-clock fields. ## Reproduce @@ -18,6 +24,14 @@ been issued. ```bash cargo test --locked --manifest-path conformance/runner/Cargo.toml cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-suite . +cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-adapters conformance/cases/core-parser.json \ + /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json +cargo run --quiet --release --locked --manifest-path conformance/runner/Cargo.toml -- \ + generate-core-differential 100000 20270803 > /tmp/core-generated.json +cargo run --quiet --release --locked --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-streams 100000 20270803 \ + /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl ``` ## Before tagging diff --git a/conformance/reports/core-differential-0.1.0-rc.1.json b/conformance/reports/core-differential-0.1.0-rc.1.json new file mode 100644 index 0000000..44194ce --- /dev/null +++ b/conformance/reports/core-differential-0.1.0-rc.1.json @@ -0,0 +1,19 @@ +{ + "format_version": "0.1-draft", + "target": "core-parser-differential", + "generator": null, + "compatible": true, + "left": { + "id": "promptsyntax-rs", + "version": "0.1.0", + "commit": "2789fac8acb810279ece887f46c025ef987ce0bf" + }, + "right": { + "id": "promptsyntax-ts", + "version": "0.1.0", + "commit": "3c2c505d6f8fa836aa42abe12c0b466bcc3ba254" + }, + "case_count": 101, + "matched_case_count": 101, + "diagnostics": [] +} diff --git a/conformance/reports/core-generated-differential-0.1.0-rc.1.json b/conformance/reports/core-generated-differential-0.1.0-rc.1.json new file mode 100644 index 0000000..1cef5ef --- /dev/null +++ b/conformance/reports/core-generated-differential-0.1.0-rc.1.json @@ -0,0 +1,23 @@ +{ + "format_version": "0.1-draft", + "target": "core-parser-differential", + "generator": { + "id": "promptsyntax-core-grammar", + "version": "0.1-draft", + "seed": "20270803" + }, + "compatible": true, + "left": { + "id": "promptsyntax-rs", + "version": "0.1.0", + "commit": "2789fac8acb810279ece887f46c025ef987ce0bf" + }, + "right": { + "id": "promptsyntax-ts", + "version": "0.1.0", + "commit": "3c2c505d6f8fa836aa42abe12c0b466bcc3ba254" + }, + "case_count": 100000, + "matched_case_count": 100000, + "diagnostics": [] +} diff --git a/conformance/reports/trace-producer-rs-0.1.0-rc.1.json b/conformance/reports/trace-producer-rs-0.1.0-rc.1.json new file mode 100644 index 0000000..1c47dfc --- /dev/null +++ b/conformance/reports/trace-producer-rs-0.1.0-rc.1.json @@ -0,0 +1,664 @@ +{ + "format_version": "0.1-draft", + "target": "trace-producer", + "implementation": { + "id": "promptsyntax-rs", + "version": "0.1.0", + "commit": "2789fac8acb810279ece887f46c025ef987ce0bf" + }, + "results": [ + { + "case_id": "producer-kept", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "profile": "trace-user-0-3-draft", + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-kept-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-4@2026-08-01", + "measured": { + "cost_usd": 0.041, + "wall_time_ms": 3210 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "bound": "model:example/atlas-4@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-4@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "status": "kept" + }, + { + "applied": 0.2, + "kind": "parameter", + "name": "temperature", + "policy": "strict", + "requested": 0.2, + "status": "applied" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the report." + }, + "i": 0, + "id": "inference-kept-001:segment:0", + "origin": "user-authored", + "step": "inference-kept-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-kept-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-kept-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example", + "vendor_profile": "vendor-profile/example-2026-08" + } + } + }, + { + "case_id": "producer-fallback", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-fallback-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + }, + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-mini@2026-08-01", + "ref": "@model:example/atlas-mini@2026-08-01", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "outcome": "failed", + "reasons": [ + "ENTITY_UNAVAILABLE" + ], + "ref": "@model:example/atlas-4@2026-08-01!" + }, + { + "bound": "model:example/atlas-mini@2026-08-01", + "measured": { + "cost_usd": 0.009, + "wall_time_ms": 1410 + }, + "outcome": "filled", + "ref": "@model:example/atlas-mini@2026-08-01" + } + ], + "bound": "model:example/atlas-mini@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-mini@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "route_step": 1, + "status": "fallback" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01! else @model:example/atlas-mini@2026-08-01" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the customer notes." + }, + "i": 0, + "id": "inference-fallback-001:segment:0", + "origin": "user-authored", + "step": "inference-fallback-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-fallback-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-fallback-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-substituted", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-substituted-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference" + }, + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-mini@2026-08-01", + "ref": "model:example/atlas-mini@2026-08-01", + "rule": "venue-best-effort-substitution" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-mini@2026-08-01", + "measured": { + "cost_usd": 0.008, + "wall_time_ms": 1330 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01" + } + ], + "bound": "model:example/atlas-mini@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-mini@2026-08-01", + "deciding_authority": "venue-operations", + "kind": "entity", + "policy": "best-effort", + "reason": "ENTITY_SUBSTITUTED", + "requested": "model:example/atlas-4@2026-08-01", + "status": "substituted" + } + ], + "outcome": "invoked", + "policy": "best-effort", + "requested": "@model:example/atlas-4@2026-08-01" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the risk report." + }, + "i": 0, + "id": "inference-substituted-001:segment:0", + "origin": "user-authored", + "step": "inference-substituted-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-substituted-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-substituted-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-refused", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-refused-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [ + { + "authority": "venue-operations", + "reason": "FALLBACK_EXHAUSTED", + "recourse": { + "available": true, + "kind": "retry" + }, + "ref": "@model:example/atlas-4@2026-08-01!" + } + ] + }, + "routing": { + "attempts": [ + { + "outcome": "failed", + "reasons": [ + "ENTITY_UNAVAILABLE" + ], + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "fill": [ + { + "deciding_authority": "venue-operations", + "kind": "entity", + "policy": "strict", + "reason": "FALLBACK_EXHAUSTED", + "requested": "model:example/atlas-4@2026-08-01", + "status": "refused" + } + ], + "outcome": "refused", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the restricted report." + }, + "i": 0, + "id": "inference-refused-001:segment:0", + "origin": "user-authored", + "step": "inference-refused-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-refused-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-refused-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-boundary-failed", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-boundary-failed-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-4@2026-08-01", + "measured": { + "cost_usd": 0.017, + "wall_time_ms": 2010 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "bound": "model:example/atlas-4@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-4@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "status": "kept" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Find the latest customer note." + }, + "i": 0, + "id": "inference-boundary-failed-001:segment:0", + "origin": "user-authored", + "step": "inference-boundary-failed-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-boundary-failed-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + }, + { + "event": "boundary", + "fill": [ + { + "deciding_authority": "venue-operations", + "kind": "action", + "reason": "ENTITY_UNAVAILABLE", + "requested": "tool:example/search@1", + "status": "refused" + } + ], + "id": "boundary-failed-001", + "measured": { + "wall_time_ms": 5 + }, + "outcome": "failed", + "reason": "ENTITY_UNAVAILABLE", + "sent": { + "state": "inline", + "text": "latest customer note" + }, + "tool": "tool:example/search@1" + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-boundary-failed-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-boundary-succeeded", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-boundary-succeeded-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-4@2026-08-01", + "measured": { + "cost_usd": 0.017, + "wall_time_ms": 2010 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "bound": "model:example/atlas-4@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-4@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "status": "kept" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Find the latest customer note." + }, + "i": 0, + "id": "inference-boundary-succeeded-001:segment:0", + "origin": "user-authored", + "step": "inference-boundary-succeeded-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-boundary-succeeded-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + }, + { + "event": "boundary", + "fill": [ + { + "applied": "tool:example/search@1", + "kind": "action", + "requested": "tool:example/search@1", + "status": "applied" + } + ], + "id": "boundary-succeeded-001", + "internal": "outside-trace-perimeter", + "measured": { + "wall_time_ms": 18 + }, + "outcome": "succeeded", + "received": { + "state": "inline", + "text": "Customer requested export controls." + }, + "sent": { + "state": "inline", + "text": "latest customer note" + }, + "tool": "tool:example/search@1" + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-boundary-succeeded-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-strict-substitution-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_SUBSTITUTION_NOT_AUTHORIZED" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_SUBSTITUTION_NOT_AUTHORIZED", + "message": "an applied entity outside the authored route requires best-effort policy", + "pointer": "/inferences/0/routing/attempts/0/bound" + }, + "format_version": "0.1-draft" + } + }, + { + "case_id": "producer-multiple-filled-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_MULTIPLE_FILLED" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_MULTIPLE_FILLED", + "message": "routing facts contain more than one filled attempt", + "pointer": "/inferences/0/routing/attempts" + }, + "format_version": "0.1-draft" + } + }, + { + "case_id": "producer-missing-refusal-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_REFUSAL_REQUIRED" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_REFUSAL_REQUIRED", + "message": "routing with no filled attempt requires a refusal fact", + "pointer": "/inferences/0/routing/refusal" + }, + "format_version": "0.1-draft" + } + }, + { + "case_id": "producer-refusal-conflict-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_REFUSAL_CONFLICT" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_REFUSAL_CONFLICT", + "message": "routing cannot contain both a filled attempt and a refusal fact", + "pointer": "/inferences/0/routing/refusal" + }, + "format_version": "0.1-draft" + } + } + ] +} diff --git a/conformance/reports/trace-producer-ts-0.1.0-rc.1.json b/conformance/reports/trace-producer-ts-0.1.0-rc.1.json new file mode 100644 index 0000000..92f4f69 --- /dev/null +++ b/conformance/reports/trace-producer-ts-0.1.0-rc.1.json @@ -0,0 +1,664 @@ +{ + "format_version": "0.1-draft", + "target": "trace-producer", + "implementation": { + "id": "promptsyntax-ts", + "version": "0.1.0", + "commit": "3c2c505d6f8fa836aa42abe12c0b466bcc3ba254" + }, + "results": [ + { + "case_id": "producer-kept", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "profile": "trace-user-0-3-draft", + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-kept-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-4@2026-08-01", + "measured": { + "cost_usd": 0.041, + "wall_time_ms": 3210 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "bound": "model:example/atlas-4@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-4@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "status": "kept" + }, + { + "applied": 0.2, + "kind": "parameter", + "name": "temperature", + "policy": "strict", + "requested": 0.2, + "status": "applied" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the report." + }, + "i": 0, + "id": "inference-kept-001:segment:0", + "origin": "user-authored", + "step": "inference-kept-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-kept-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-kept-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example", + "vendor_profile": "vendor-profile/example-2026-08" + } + } + }, + { + "case_id": "producer-fallback", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-fallback-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + }, + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-mini@2026-08-01", + "ref": "@model:example/atlas-mini@2026-08-01", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "outcome": "failed", + "reasons": [ + "ENTITY_UNAVAILABLE" + ], + "ref": "@model:example/atlas-4@2026-08-01!" + }, + { + "bound": "model:example/atlas-mini@2026-08-01", + "measured": { + "cost_usd": 0.009, + "wall_time_ms": 1410 + }, + "outcome": "filled", + "ref": "@model:example/atlas-mini@2026-08-01" + } + ], + "bound": "model:example/atlas-mini@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-mini@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "route_step": 1, + "status": "fallback" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01! else @model:example/atlas-mini@2026-08-01" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the customer notes." + }, + "i": 0, + "id": "inference-fallback-001:segment:0", + "origin": "user-authored", + "step": "inference-fallback-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-fallback-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-fallback-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-substituted", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-substituted-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference" + }, + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-mini@2026-08-01", + "ref": "model:example/atlas-mini@2026-08-01", + "rule": "venue-best-effort-substitution" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-mini@2026-08-01", + "measured": { + "cost_usd": 0.008, + "wall_time_ms": 1330 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01" + } + ], + "bound": "model:example/atlas-mini@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-mini@2026-08-01", + "deciding_authority": "venue-operations", + "kind": "entity", + "policy": "best-effort", + "reason": "ENTITY_SUBSTITUTED", + "requested": "model:example/atlas-4@2026-08-01", + "status": "substituted" + } + ], + "outcome": "invoked", + "policy": "best-effort", + "requested": "@model:example/atlas-4@2026-08-01" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the risk report." + }, + "i": 0, + "id": "inference-substituted-001:segment:0", + "origin": "user-authored", + "step": "inference-substituted-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-substituted-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-substituted-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-refused", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-refused-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [ + { + "authority": "venue-operations", + "reason": "FALLBACK_EXHAUSTED", + "recourse": { + "available": true, + "kind": "retry" + }, + "ref": "@model:example/atlas-4@2026-08-01!" + } + ] + }, + "routing": { + "attempts": [ + { + "outcome": "failed", + "reasons": [ + "ENTITY_UNAVAILABLE" + ], + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "fill": [ + { + "deciding_authority": "venue-operations", + "kind": "entity", + "policy": "strict", + "reason": "FALLBACK_EXHAUSTED", + "requested": "model:example/atlas-4@2026-08-01", + "status": "refused" + } + ], + "outcome": "refused", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Summarize the restricted report." + }, + "i": 0, + "id": "inference-refused-001:segment:0", + "origin": "user-authored", + "step": "inference-refused-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-refused-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-refused-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-boundary-failed", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-boundary-failed-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-4@2026-08-01", + "measured": { + "cost_usd": 0.017, + "wall_time_ms": 2010 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "bound": "model:example/atlas-4@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-4@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "status": "kept" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Find the latest customer note." + }, + "i": 0, + "id": "inference-boundary-failed-001:segment:0", + "origin": "user-authored", + "step": "inference-boundary-failed-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-boundary-failed-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + }, + { + "event": "boundary", + "fill": [ + { + "deciding_authority": "venue-operations", + "kind": "action", + "reason": "ENTITY_UNAVAILABLE", + "requested": "tool:example/search@1", + "status": "refused" + } + ], + "id": "boundary-failed-001", + "measured": { + "wall_time_ms": 5 + }, + "outcome": "failed", + "reason": "ENTITY_UNAVAILABLE", + "sent": { + "state": "inline", + "text": "latest customer note" + }, + "tool": "tool:example/search@1" + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-boundary-failed-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-boundary-succeeded", + "conformant": true, + "diagnostics": [], + "output": { + "content_mode": "tiered", + "coverage": { + "assembly": "user-initiated-only", + "inline_threshold_bytes": 4096, + "routing": "complete" + }, + "events": [ + { + "event": "inference", + "id": "inference-boundary-succeeded-001", + "kind": "user-initiated", + "resolution": { + "bindings": [ + { + "ambiguity_surfaced": false, + "bound": "model:example/atlas-4@2026-08-01", + "ref": "@model:example/atlas-4@2026-08-01!", + "rule": "qualified-canonical-reference" + } + ], + "refusals": [] + }, + "routing": { + "attempts": [ + { + "bound": "model:example/atlas-4@2026-08-01", + "measured": { + "cost_usd": 0.017, + "wall_time_ms": 2010 + }, + "outcome": "filled", + "ref": "@model:example/atlas-4@2026-08-01!" + } + ], + "bound": "model:example/atlas-4@2026-08-01", + "fill": [ + { + "applied": "model:example/atlas-4@2026-08-01", + "kind": "entity", + "policy": "strict", + "requested": "model:example/atlas-4@2026-08-01", + "status": "kept" + } + ], + "outcome": "invoked", + "policy": "strict", + "requested": "@model:example/atlas-4@2026-08-01!" + }, + "segments": [ + { + "content": { + "state": "inline", + "text": "Find the latest customer note." + }, + "i": 0, + "id": "inference-boundary-succeeded-001:segment:0", + "origin": "user-authored", + "step": "inference-boundary-succeeded-001:step:user-content" + } + ], + "steps": [ + { + "id": "inference-boundary-succeeded-001:step:user-content", + "kind": "user-content", + "tier": "user" + } + ] + }, + { + "event": "boundary", + "fill": [ + { + "applied": "tool:example/search@1", + "kind": "action", + "requested": "tool:example/search@1", + "status": "applied" + } + ], + "id": "boundary-succeeded-001", + "internal": "outside-trace-perimeter", + "measured": { + "wall_time_ms": 18 + }, + "outcome": "succeeded", + "received": { + "state": "inline", + "text": "Customer requested export controls." + }, + "sent": { + "state": "inline", + "text": "latest customer note" + }, + "tool": "tool:example/search@1" + } + ], + "mode": "executed", + "ps_trace": "0.3-draft", + "tier": "user", + "turn": "turn-producer-boundary-succeeded-001", + "venue": { + "capability_document": "https://venue.example/ps-capabilities/0.3.json", + "id": "venue:example" + } + } + }, + { + "case_id": "producer-strict-substitution-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_SUBSTITUTION_NOT_AUTHORIZED" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_SUBSTITUTION_NOT_AUTHORIZED", + "message": "an applied entity outside the authored route requires best-effort policy", + "pointer": "/inferences/0/routing/attempts/0/bound" + }, + "format_version": "0.1-draft" + } + }, + { + "case_id": "producer-multiple-filled-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_MULTIPLE_FILLED" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_MULTIPLE_FILLED", + "message": "routing facts contain more than one filled attempt", + "pointer": "/inferences/0/routing/attempts" + }, + "format_version": "0.1-draft" + } + }, + { + "case_id": "producer-missing-refusal-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_REFUSAL_REQUIRED" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_REFUSAL_REQUIRED", + "message": "routing with no filled attempt requires a refusal fact", + "pointer": "/inferences/0/routing/refusal" + }, + "format_version": "0.1-draft" + } + }, + { + "case_id": "producer-refusal-conflict-rejected", + "conformant": true, + "diagnostics": [ + "TRACE_PRODUCER_REFUSAL_CONFLICT" + ], + "output": { + "error": { + "code": "TRACE_PRODUCER_REFUSAL_CONFLICT", + "message": "routing cannot contain both a filled attempt and a refusal fact", + "pointer": "/inferences/0/routing/refusal" + }, + "format_version": "0.1-draft" + } + } + ] +} diff --git a/conformance/runner/README.md b/conformance/runner/README.md index 2a39a0c..5595dcb 100644 --- a/conformance/runner/README.md +++ b/conformance/runner/README.md @@ -18,6 +18,21 @@ cargo run --manifest-path conformance/runner/Cargo.toml -- \ examples/prompt-trace-0.3-draft.example.json cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . + +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-adapters conformance/cases/core-parser.json \ + /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json + +cargo run --release --manifest-path conformance/runner/Cargo.toml -- \ + generate-core-differential 100000 20270803 > /tmp/core-generated.json + +cargo run --release --manifest-path conformance/runner/Cargo.toml -- \ + compare-core-streams 100000 20270803 \ + /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl + +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + run-trace-producer-adapter . promptsyntax-rs 0.1.0 \ + /path/to/ps-trace-producer ``` The command emits deterministic JSON and exits with: @@ -26,6 +41,22 @@ The command emits deterministic JSON and exits with: - `1` when the input produces conformance diagnostics; or - `2` for invalid command usage or an internal report-serialization error. +`compare-core-adapters` requires two already-produced adapter result documents. It checks +that both ran every canonical case, satisfied the specification-owned expected fields, and +emitted equivalent normalized JSON values for every complete parser output. Implementation +metadata may differ. Case outputs may not. + +`generate-core-differential` deterministically constructs syntax from 32 grammar families +without embedding expected parser results. `compare-core-streams` validates the exact +generated ID sequence and compares full normalized results while retaining only one result +from each implementation in memory. The report records the generator version and seed. + +`run-trace-producer-adapter` executes every standard-owned producer case. It validates each +input before invoking the implementation, withholds the independent transcript from the +producer, then validates the emitted Trace against both the Trace schema and transcript. +Expected producer rejections must use the registered JSON error envelope and exit status +`1`. + The runner performs no network requests and needs no provider SDK, model credentials, -Python, Node, or Bun. It is bootstrap tooling for a working draft, not a certification -utility. +Python, Node, or Bun. Producing the TypeScript adapter document separately requires Bun. +This is bootstrap tooling for a working draft, not a certification utility. diff --git a/conformance/runner/src/core.rs b/conformance/runner/src/core.rs new file mode 100644 index 0000000..759f8d7 --- /dev/null +++ b/conformance/runner/src/core.rs @@ -0,0 +1,740 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{Diagnostic, GeneratorMetadata}; + +const SUPPORTED_FORMAT: &str = "0.1-draft"; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct CoreDifferentialReport { + pub format_version: String, + pub target: String, + pub generator: Option, + pub compatible: bool, + pub left: Option, + pub right: Option, + pub case_count: usize, + pub matched_case_count: usize, + pub diagnostics: Vec, +} + +impl CoreDifferentialReport { + #[must_use] + pub fn invalid_input(code: &str, message: impl Into) -> Self { + Self { + format_version: SUPPORTED_FORMAT.to_owned(), + target: "core-parser-differential".to_owned(), + generator: None, + compatible: false, + left: None, + right: None, + case_count: 0, + matched_case_count: 0, + diagnostics: vec![Diagnostic { + code: code.to_owned(), + pointer: String::new(), + message: message.into(), + }], + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Implementation { + pub id: String, + pub version: String, + pub commit: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct AdapterResult { + format_version: String, + target: String, + implementation: Implementation, + results: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct AdapterStreamHeader { + format_version: String, + target: String, + implementation: Implementation, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CaseResult { + case_id: String, + conformant: bool, + diagnostics: Vec, + output: Value, +} + +#[derive(Debug, Deserialize)] +struct CanonicalCase { + id: String, +} + +#[must_use] +#[allow(clippy::too_many_lines)] +pub fn compare_core_adapters_json( + cases_input: &[u8], + left_input: &[u8], + right_input: &[u8], +) -> CoreDifferentialReport { + let cases = match serde_json::from_slice::>(cases_input) { + Ok(cases) => cases, + Err(error) => { + return CoreDifferentialReport::invalid_input( + "CORE_CASES_JSON_INVALID", + error.to_string(), + ); + } + }; + let left = match serde_json::from_slice::(left_input) { + Ok(result) => result, + Err(error) => { + return CoreDifferentialReport::invalid_input( + "LEFT_ADAPTER_JSON_INVALID", + error.to_string(), + ); + } + }; + let right = match serde_json::from_slice::(right_input) { + Ok(result) => result, + Err(error) => { + return CoreDifferentialReport::invalid_input( + "RIGHT_ADAPTER_JSON_INVALID", + error.to_string(), + ); + } + }; + + let left_implementation = left.implementation.clone(); + let right_implementation = right.implementation.clone(); + let mut diagnostics = Vec::new(); + let canonical_ids = index_canonical_cases(cases, &mut diagnostics); + check_envelope(&left, "left", &mut diagnostics); + check_envelope(&right, "right", &mut diagnostics); + if left.implementation == right.implementation { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_IMPLEMENTATIONS_IDENTICAL".to_owned(), + pointer: "/implementation".to_owned(), + message: "differential comparison requires two distinct implementations".to_owned(), + }); + } + + let left_results = index_results(left.results, "left", &mut diagnostics); + let right_results = index_results(right.results, "right", &mut diagnostics); + let mut matched_case_count = 0; + + for case_id in &canonical_ids { + let pointer = format!("/results/{case_id}"); + let left_case = left_results.get(case_id); + let right_case = right_results.get(case_id); + if left_case.is_none() { + diagnostics.push(Diagnostic { + code: "CORE_CASE_MISSING_LEFT".to_owned(), + pointer: pointer.clone(), + message: format!("canonical case {case_id} is absent from the left adapter output"), + }); + } + if right_case.is_none() { + diagnostics.push(Diagnostic { + code: "CORE_CASE_MISSING_RIGHT".to_owned(), + pointer: pointer.clone(), + message: format!( + "canonical case {case_id} is absent from the right adapter output" + ), + }); + } + let (Some(left_case), Some(right_case)) = (left_case, right_case) else { + continue; + }; + if !left_case.conformant { + diagnostics.push(Diagnostic { + code: "CORE_CASE_NONCONFORMANT_LEFT".to_owned(), + pointer: pointer.clone(), + message: format!( + "left adapter rejected {case_id}: {}", + left_case.diagnostics.join(", ") + ), + }); + } + if !right_case.conformant { + diagnostics.push(Diagnostic { + code: "CORE_CASE_NONCONFORMANT_RIGHT".to_owned(), + pointer: pointer.clone(), + message: format!( + "right adapter rejected {case_id}: {}", + right_case.diagnostics.join(", ") + ), + }); + } + if left_case.output != right_case.output { + diagnostics.push(Diagnostic { + code: "CORE_OUTPUT_DIFFERENTIAL".to_owned(), + pointer, + message: format!("normalized parser outputs differ for case {case_id}"), + }); + continue; + } + matched_case_count += 1; + } + + for case_id in left_results.keys() { + if !canonical_ids.contains(case_id) { + diagnostics.push(Diagnostic { + code: "CORE_CASE_UNEXPECTED_LEFT".to_owned(), + pointer: format!("/results/{case_id}"), + message: format!("left adapter emitted non-canonical case {case_id}"), + }); + } + } + for case_id in right_results.keys() { + if !canonical_ids.contains(case_id) { + diagnostics.push(Diagnostic { + code: "CORE_CASE_UNEXPECTED_RIGHT".to_owned(), + pointer: format!("/results/{case_id}"), + message: format!("right adapter emitted non-canonical case {case_id}"), + }); + } + } + + let case_count = canonical_ids.len(); + CoreDifferentialReport { + format_version: SUPPORTED_FORMAT.to_owned(), + target: "core-parser-differential".to_owned(), + generator: None, + compatible: diagnostics.is_empty(), + left: Some(left_implementation), + right: Some(right_implementation), + case_count, + matched_case_count, + diagnostics, + } +} + +/// Compare two normalized JSON Lines adapter streams in bounded memory. +/// +/// # Errors +/// +/// File and JSON errors are represented in the returned non-compatible report rather than +/// returned separately. +#[must_use] +pub fn compare_core_adapter_streams( + expected_case_count: usize, + seed: u64, + left_path: &Path, + right_path: &Path, +) -> CoreDifferentialReport { + let left = match File::open(left_path) { + Ok(file) => BufReader::new(file), + Err(error) => { + return invalid_generated( + expected_case_count, + seed, + "LEFT_ADAPTER_READ_FAILED", + error.to_string(), + ); + } + }; + let right = match File::open(right_path) { + Ok(file) => BufReader::new(file), + Err(error) => { + return invalid_generated( + expected_case_count, + seed, + "RIGHT_ADAPTER_READ_FAILED", + error.to_string(), + ); + } + }; + compare_core_adapter_readers(expected_case_count, seed, left, right) +} + +#[allow(clippy::too_many_lines)] +fn compare_core_adapter_readers( + expected_case_count: usize, + seed: u64, + mut left: impl BufRead, + mut right: impl BufRead, +) -> CoreDifferentialReport { + let left_header = match read_json_line::(&mut left) { + Ok(Some(header)) => header, + Ok(None) => { + return invalid_generated( + expected_case_count, + seed, + "LEFT_ADAPTER_JSON_INVALID", + "left adapter stream has no header", + ); + } + Err(error) => { + return invalid_generated( + expected_case_count, + seed, + "LEFT_ADAPTER_JSON_INVALID", + error, + ); + } + }; + let right_header = match read_json_line::(&mut right) { + Ok(Some(header)) => header, + Ok(None) => { + return invalid_generated( + expected_case_count, + seed, + "RIGHT_ADAPTER_JSON_INVALID", + "right adapter stream has no header", + ); + } + Err(error) => { + return invalid_generated( + expected_case_count, + seed, + "RIGHT_ADAPTER_JSON_INVALID", + error, + ); + } + }; + let mut diagnostics = Vec::new(); + check_stream_header(&left_header, "left", &mut diagnostics); + check_stream_header(&right_header, "right", &mut diagnostics); + if left_header.implementation == right_header.implementation { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_IMPLEMENTATIONS_IDENTICAL".to_owned(), + pointer: "/implementation".to_owned(), + message: "differential comparison requires two distinct implementations".to_owned(), + }); + } + + let mut matched_case_count = 0; + for index in 0..expected_case_count { + let case_id = format!("core-generated-{index:06}"); + let left_case = match read_json_line::(&mut left) { + Ok(Some(case)) => Some(case), + Ok(None) => { + diagnostics.push(Diagnostic { + code: "CORE_CASE_MISSING_LEFT".to_owned(), + pointer: format!("/results/{case_id}"), + message: format!("left adapter stream ended before {case_id}"), + }); + None + } + Err(error) => { + diagnostics.push(Diagnostic { + code: "LEFT_ADAPTER_JSON_INVALID".to_owned(), + pointer: format!("/results/{case_id}"), + message: error, + }); + None + } + }; + let right_case = match read_json_line::(&mut right) { + Ok(Some(case)) => Some(case), + Ok(None) => { + diagnostics.push(Diagnostic { + code: "CORE_CASE_MISSING_RIGHT".to_owned(), + pointer: format!("/results/{case_id}"), + message: format!("right adapter stream ended before {case_id}"), + }); + None + } + Err(error) => { + diagnostics.push(Diagnostic { + code: "RIGHT_ADAPTER_JSON_INVALID".to_owned(), + pointer: format!("/results/{case_id}"), + message: error, + }); + None + } + }; + let (Some(left_case), Some(right_case)) = (left_case, right_case) else { + break; + }; + let pointer = format!("/results/{case_id}"); + let mut inventory_matches = true; + if left_case.case_id != case_id { + diagnostics.push(Diagnostic { + code: "CORE_CASE_UNEXPECTED_LEFT".to_owned(), + pointer: pointer.clone(), + message: format!( + "left adapter emitted {} where {case_id} was required", + left_case.case_id + ), + }); + inventory_matches = false; + } + if right_case.case_id != case_id { + diagnostics.push(Diagnostic { + code: "CORE_CASE_UNEXPECTED_RIGHT".to_owned(), + pointer: pointer.clone(), + message: format!( + "right adapter emitted {} where {case_id} was required", + right_case.case_id + ), + }); + inventory_matches = false; + } + if !left_case.conformant { + diagnostics.push(Diagnostic { + code: "CORE_CASE_NONCONFORMANT_LEFT".to_owned(), + pointer: pointer.clone(), + message: format!( + "left adapter rejected {case_id}: {}", + left_case.diagnostics.join(", ") + ), + }); + } + if !right_case.conformant { + diagnostics.push(Diagnostic { + code: "CORE_CASE_NONCONFORMANT_RIGHT".to_owned(), + pointer: pointer.clone(), + message: format!( + "right adapter rejected {case_id}: {}", + right_case.diagnostics.join(", ") + ), + }); + } + if left_case.output != right_case.output { + diagnostics.push(Diagnostic { + code: "CORE_OUTPUT_DIFFERENTIAL".to_owned(), + pointer, + message: format!("normalized parser outputs differ for case {case_id}"), + }); + } else if inventory_matches { + matched_case_count += 1; + } + } + + check_no_extra_line(&mut left, "left", &mut diagnostics); + check_no_extra_line(&mut right, "right", &mut diagnostics); + CoreDifferentialReport { + format_version: SUPPORTED_FORMAT.to_owned(), + target: "core-parser-differential".to_owned(), + generator: Some(GeneratorMetadata::core(seed)), + compatible: diagnostics.is_empty(), + left: Some(left_header.implementation), + right: Some(right_header.implementation), + case_count: expected_case_count, + matched_case_count, + diagnostics, + } +} + +fn invalid_generated( + expected_case_count: usize, + seed: u64, + code: &str, + message: impl Into, +) -> CoreDifferentialReport { + CoreDifferentialReport { + format_version: SUPPORTED_FORMAT.to_owned(), + target: "core-parser-differential".to_owned(), + generator: Some(GeneratorMetadata::core(seed)), + compatible: false, + left: None, + right: None, + case_count: expected_case_count, + matched_case_count: 0, + diagnostics: vec![Diagnostic { + code: code.to_owned(), + pointer: String::new(), + message: message.into(), + }], + } +} + +fn read_json_line Deserialize<'de>>( + input: &mut impl BufRead, +) -> Result, String> { + let mut line = String::new(); + let count = input + .read_line(&mut line) + .map_err(|error| error.to_string())?; + if count == 0 { + return Ok(None); + } + while matches!(line.as_bytes().last(), Some(b'\n' | b'\r')) { + line.pop(); + } + serde_json::from_str(&line) + .map(Some) + .map_err(|error| error.to_string()) +} + +fn check_stream_header( + header: &AdapterStreamHeader, + side: &str, + diagnostics: &mut Vec, +) { + if header.format_version != SUPPORTED_FORMAT { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_FORMAT_UNSUPPORTED".to_owned(), + pointer: format!("/{side}/format_version"), + message: format!("{side} adapter format must be {SUPPORTED_FORMAT}"), + }); + } + if header.target != "core-parser" { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_TARGET_INVALID".to_owned(), + pointer: format!("/{side}/target"), + message: format!("{side} adapter target must be core-parser"), + }); + } + if header.implementation.id.is_empty() + || header.implementation.version.is_empty() + || !is_commit(&header.implementation.commit) + { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_IMPLEMENTATION_INVALID".to_owned(), + pointer: format!("/{side}/implementation"), + message: format!("{side} adapter implementation metadata is invalid"), + }); + } +} + +fn check_no_extra_line(input: &mut impl BufRead, side: &str, diagnostics: &mut Vec) { + match read_json_line::(input) { + Ok(None) => {} + Ok(Some(_)) => diagnostics.push(Diagnostic { + code: format!("CORE_CASE_UNEXPECTED_{}", side.to_ascii_uppercase()), + pointer: format!("/{side}/results"), + message: format!("{side} adapter emitted more cases than the generated inventory"), + }), + Err(error) => diagnostics.push(Diagnostic { + code: format!("{}_ADAPTER_JSON_INVALID", side.to_ascii_uppercase()), + pointer: format!("/{side}/results"), + message: error, + }), + } +} + +fn index_canonical_cases( + cases: Vec, + diagnostics: &mut Vec, +) -> std::collections::BTreeSet { + let mut ids = std::collections::BTreeSet::new(); + for case in cases { + if !ids.insert(case.id.clone()) { + diagnostics.push(Diagnostic { + code: "CORE_CASES_ID_DUPLICATE".to_owned(), + pointer: format!("/cases/{}", case.id), + message: format!("canonical corpus repeats case id {}", case.id), + }); + } + } + ids +} + +fn check_envelope(result: &AdapterResult, side: &str, diagnostics: &mut Vec) { + if result.format_version != SUPPORTED_FORMAT { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_FORMAT_UNSUPPORTED".to_owned(), + pointer: format!("/{side}/format_version"), + message: format!("{side} adapter format must be {SUPPORTED_FORMAT}"), + }); + } + if result.target != "core-parser" { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_TARGET_INVALID".to_owned(), + pointer: format!("/{side}/target"), + message: format!("{side} adapter target must be core-parser"), + }); + } + if result.implementation.id.is_empty() + || result.implementation.version.is_empty() + || !is_commit(&result.implementation.commit) + { + diagnostics.push(Diagnostic { + code: "CORE_ADAPTER_IMPLEMENTATION_INVALID".to_owned(), + pointer: format!("/{side}/implementation"), + message: format!("{side} adapter implementation metadata is invalid"), + }); + } +} + +fn index_results( + results: Vec, + side: &str, + diagnostics: &mut Vec, +) -> BTreeMap { + let mut indexed = BTreeMap::new(); + for result in results { + let case_id = result.case_id.clone(); + if indexed.insert(case_id.clone(), result).is_some() { + diagnostics.push(Diagnostic { + code: "CORE_CASE_ID_DUPLICATE".to_owned(), + pointer: format!("/{side}/results/{case_id}"), + message: format!("{side} adapter repeats case id {case_id}"), + }); + } + } + indexed +} + +fn is_commit(value: &str) -> bool { + value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use serde_json::json; + + use super::{compare_core_adapter_readers, compare_core_adapters_json}; + + fn cases(ids: &[&str]) -> Vec { + serde_json::to_vec(&ids.iter().map(|id| json!({ "id": id })).collect::>()) + .expect("cases JSON") + } + + #[allow(clippy::needless_pass_by_value)] + fn adapter(id: &str, results: serde_json::Value) -> Vec { + serde_json::to_vec(&json!({ + "format_version": "0.1-draft", + "target": "core-parser", + "implementation": { + "id": id, + "version": "0.1.0", + "commit": "0123456789abcdef0123456789abcdef01234567" + }, + "results": results + })) + .expect("adapter JSON") + } + + #[allow(clippy::needless_pass_by_value)] + fn case(id: &str, output: serde_json::Value) -> serde_json::Value { + json!({ + "case_id": id, + "conformant": true, + "diagnostics": [], + "output": output + }) + } + + fn stream(id: &str, cases: &[serde_json::Value]) -> Cursor> { + let mut lines = vec![json!({ + "format_version": "0.1-draft", + "target": "core-parser", + "implementation": { + "id": id, + "version": "0.1.0", + "commit": "0123456789abcdef0123456789abcdef01234567" + } + })]; + lines.extend_from_slice(cases); + let text = lines + .into_iter() + .map(|line| serde_json::to_string(&line).expect("stream line")) + .collect::>() + .join("\n") + + "\n"; + Cursor::new(text.into_bytes()) + } + + #[test] + fn matching_outputs_are_compatible() { + let left = adapter("rust", json!([case("one", json!({"kind": "text"}))])); + let right = adapter("typescript", json!([case("one", json!({"kind": "text"}))])); + let report = compare_core_adapters_json(&cases(&["one"]), &left, &right); + assert!(report.compatible); + assert_eq!(report.case_count, 1); + assert_eq!(report.matched_case_count, 1); + } + + #[test] + fn mismatched_outputs_are_reported() { + let left = adapter("rust", json!([case("one", json!({"kind": "text"}))])); + let right = adapter( + "typescript", + json!([case("one", json!({"kind": "directive"}))]), + ); + let report = compare_core_adapters_json(&cases(&["one"]), &left, &right); + assert!(!report.compatible); + assert_eq!(report.matched_case_count, 0); + assert_eq!(report.diagnostics[0].code, "CORE_OUTPUT_DIFFERENTIAL"); + } + + #[test] + fn missing_and_duplicate_cases_are_reported() { + let left = adapter( + "rust", + json!([case("one", json!(1)), case("one", json!(1))]), + ); + let right = adapter("typescript", json!([case("two", json!(2))])); + let report = compare_core_adapters_json(&cases(&["one", "two"]), &left, &right); + assert!(!report.compatible); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORE_CASE_ID_DUPLICATE") + ); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORE_CASE_MISSING_LEFT") + ); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORE_CASE_MISSING_RIGHT") + ); + } + + #[test] + fn both_adapters_cannot_skip_the_same_canonical_case() { + let left = adapter("rust", json!([])); + let right = adapter("typescript", json!([])); + let report = compare_core_adapters_json(&cases(&["one"]), &left, &right); + assert!(!report.compatible); + assert_eq!(report.case_count, 1); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORE_CASE_MISSING_LEFT") + ); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "CORE_CASE_MISSING_RIGHT") + ); + } + + #[test] + fn generated_streams_match_in_bounded_memory() { + let cases = [ + case("core-generated-000000", json!({"kind": "text"})), + case("core-generated-000001", json!({"kind": "reference"})), + ]; + let report = compare_core_adapter_readers( + 2, + 20_270_803, + stream("rust", &cases), + stream("typescript", &cases), + ); + assert!(report.compatible); + assert_eq!(report.case_count, 2); + assert_eq!(report.matched_case_count, 2); + assert_eq!( + report.generator.as_ref().map(|generator| &generator.seed), + Some(&"20270803".to_owned()) + ); + } +} diff --git a/conformance/runner/src/generator.rs b/conformance/runner/src/generator.rs new file mode 100644 index 0000000..b7c5c5e --- /dev/null +++ b/conformance/runner/src/generator.rs @@ -0,0 +1,330 @@ +use std::io::Write; + +use serde::Serialize; +use serde::ser::{SerializeSeq, Serializer as _}; + +const FORMAT_VERSION: &str = "0.1-draft"; +const GENERATOR_ID: &str = "promptsyntax-core-grammar"; + +#[derive(Debug, Serialize)] +struct GeneratedCase { + id: String, + name: String, + source: String, + options: Options, +} + +#[derive(Debug, Default, Serialize)] +struct Options { + #[serde(skip_serializing_if = "Vec::is_empty")] + entities: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + actions: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + authoring_namespaces: Vec, +} + +/// Metadata embedded in generated-differential reports. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct GeneratorMetadata { + pub id: String, + pub version: String, + pub seed: String, +} + +impl GeneratorMetadata { + #[must_use] + pub fn core(seed: u64) -> Self { + Self { + id: GENERATOR_ID.to_owned(), + version: FORMAT_VERSION.to_owned(), + seed: seed.to_string(), + } + } +} + +/// Write deterministic grammar-generated Core inputs as one JSON array. +/// +/// The generator supplies syntax and capability environments, but deliberately supplies no +/// expected parser answer. Differential comparison therefore cannot inherit Rust parser +/// behavior as an oracle. +/// +/// # Errors +/// +/// Returns an error if serialization or writing fails. +pub fn write_generated_core_cases( + output: impl Write, + case_count: usize, + seed: u64, +) -> Result<(), String> { + let mut serializer = serde_json::Serializer::new(output); + let mut sequence = serializer + .serialize_seq(Some(case_count)) + .map_err(|error| error.to_string())?; + let mut random = SplitMix64::new(seed); + for index in 0..case_count { + sequence + .serialize_element(&generate_case(index, &mut random)) + .map_err(|error| error.to_string())?; + } + sequence.end().map_err(|error| error.to_string()) +} + +#[allow(clippy::too_many_lines)] +fn generate_case(index: usize, random: &mut SplitMix64) -> GeneratedCase { + let nonce = random.next(); + let family = index % 32; + let model = pick(&["atlas-4", "atlas-mini", "café", "អាត្លាស"], nonce); + let file = pick( + &["notes.md", "q3-report.md", "数据.csv", "របាយការណ៍.md"], + nonce >> 3, + ); + let prose = pick( + &["Summarize", "សង្ខេប", "要約", "ملخص", "סיכום", "สรุป"], + nonce >> 7, + ); + let spacing = pick(&[" ", "\t", "\n", " "], nonce >> 11); + let number = nonce % 10_000; + let (name, source, options) = match family { + 0 => ( + "plain-unicode", + format!("{prose}{spacing}plain text {number}"), + Options::default(), + ), + 1 => ( + "qualified-reference", + format!("{prose}{spacing}@model:atlas/{model}"), + Options::default(), + ), + 2 => ( + "file-reference", + format!("Read{spacing}@file:{file}"), + Options::default(), + ), + 3 => ( + "declared-bare-entity", + format!("@{model}{spacing}{prose}"), + Options { + entities: vec![model.to_owned()], + ..Options::default() + }, + ), + 4 => ( + "declared-action", + format!("/{prose}(count: {number})"), + Options { + actions: vec![prose.to_owned()], + ..Options::default() + }, + ), + 5 => ( + "full-width-action", + format!("／{prose}(enabled: true)"), + Options { + actions: vec![prose.to_owned()], + ..Options::default() + }, + ), + 6 => ( + "reference-arguments", + format!( + "@tool:search(query: \"{prose} {number}\", count: {}, cache: null)", + number % 17 + ), + Options::default(), + ), + 7 => ( + "escaped-argument", + format!("@tool:search(query: \"item \\\"{number}\\\", next\")"), + Options::default(), + ), + 8 => ( + "fallback-route", + format!( + "@model:atlas/atlas-4 limit(wall_time: {}ms) else @model:atlas/atlas-mini else ask", + 1 + number + ), + Options::default(), + ), + 9 => ( + "strict-fail-route", + "@model:atlas/atlas-4! else @model:atlas/atlas-mini else fail".to_owned(), + Options::default(), + ), + 10 => ( + "normal-span", + format!("{prose} {number}"), + Options::default(), + ), + 11 => ( + "nested-span", + format!("Outer {prose} end"), + Options::default(), + ), + 12 => ( + "span-attributes", + format!("{prose}"), + Options::default(), + ), + 13 => ( + "authoring-segment", + format!(""), + Options { + authoring_namespaces: vec!["agency".to_owned()], + ..Options::default() + }, + ), + 14 => ( + "backtick-fence", + format!("```ps\n@model:atlas/{model}\n/{prose}\n```"), + Options { + actions: vec![prose.to_owned()], + ..Options::default() + }, + ), + 15 => ( + "tilde-fence", + format!("~~~text\n@file:{file}\n~~~"), + Options::default(), + ), + 16 => ( + "escaped-sigils", + format!("\\@model:atlas/{model}{spacing}\\/{prose}"), + Options { + actions: vec![prose.to_owned()], + ..Options::default() + }, + ), + 17 => ( + "url-email-path", + format!( + "https://user{number}@example.test/@model:atlas/{model} mail{number}@example.test /usr/bin" + ), + Options::default(), + ), + 18 => ( + "foreign-markup", + format!(" @model:atlas/{model}\">{prose}"), + Options::default(), + ), + 19 => ( + "markup-comment", + format!(""), + Options::default(), + ), + 20 => ( + "malformed-reference", + format!("@model:{spacing}{model}"), + Options::default(), + ), + 21 => ( + "malformed-reference-arguments", + format!("@tool:search(query{number})"), + Options::default(), + ), + 22 => ( + "malformed-declared-action", + format!("/{prose}(value: \"{number}\""), + Options { + actions: vec![prose.to_owned()], + ..Options::default() + }, + ), + 23 => ( + "bidi-reference", + format!("@model:atlas/atlas-4\u{2066}{number}"), + Options::default(), + ), + 24 => ( + "bidi-argument", + format!("@tool:search(query: \"item {number}\u{202e}\")"), + Options::default(), + ), + 25 => ( + "no-spacing-boundary", + format!("{prose}@model:atlas/{model}"), + Options::default(), + ), + 26 => ( + "punctuation-boundary", + format!("({prose}: @file:{file}), case {number}."), + Options::default(), + ), + 27 => ( + "frontmatter-lf", + format!("---ps\ncase: \"{number}\"\n---\n{prose}"), + Options::default(), + ), + 28 => ( + "frontmatter-crlf", + format!("---ps\r\ncase: \"{number}\"\r\n---\r\n{prose}"), + Options::default(), + ), + 29 => ( + "unclosed-span", + format!("Before {prose} {number}"), + Options::default(), + ), + 30 => ( + "mixed-directives", + format!("@model:atlas/{model}{spacing}/{prose}(to: \"km\"){spacing}@file:{file}"), + Options { + actions: vec![prose.to_owned()], + ..Options::default() + }, + ), + _ => ( + "mixed-line-endings", + format!("{prose}\r\n@model:atlas/{model}\ncase {number}"), + Options::default(), + ), + }; + GeneratedCase { + id: format!("core-generated-{index:06}"), + name: name.to_owned(), + source, + options, + } +} + +fn pick<'a>(values: &'a [&str], value: u64) -> &'a str { + let index = usize::try_from(value % values.len() as u64).expect("small choice index"); + values[index] +} + +struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + const fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut value = self.state; + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) + } +} + +#[cfg(test)] +mod tests { + use super::write_generated_core_cases; + + #[test] + fn generation_is_deterministic_and_contains_no_expected_answers() { + let mut first = Vec::new(); + let mut second = Vec::new(); + write_generated_core_cases(&mut first, 64, 20_270_803).expect("first generation"); + write_generated_core_cases(&mut second, 64, 20_270_803).expect("second generation"); + assert_eq!(first, second); + let cases: serde_json::Value = serde_json::from_slice(&first).expect("generated JSON"); + assert_eq!(cases.as_array().map(Vec::len), Some(64)); + assert!(cases[0].get("data_plane").is_none()); + assert!(cases[0].get("directives").is_none()); + assert!(cases[0].get("diagnostics").is_none()); + } +} diff --git a/conformance/runner/src/lib.rs b/conformance/runner/src/lib.rs index 57a33fa..ad073a7 100644 --- a/conformance/runner/src/lib.rs +++ b/conformance/runner/src/lib.rs @@ -7,9 +7,15 @@ use jsonschema::PatternOptions; use serde::{Deserialize, Serialize}; use serde_json::Value; +mod core; +mod generator; +mod producer; mod suite; mod trace; +pub use core::{CoreDifferentialReport, compare_core_adapter_streams, compare_core_adapters_json}; +pub use generator::{GeneratorMetadata, write_generated_core_cases}; +pub use producer::{TraceProducerAdapterReport, run_trace_producer_adapter}; pub use suite::{SuiteReport, run_suite}; const SUPPORTED_FORMAT: &str = "0.1-draft"; @@ -302,6 +308,58 @@ pub fn validate_instance_json(schema_input: &[u8], instance_input: &[u8]) -> Val } } +#[must_use] +pub fn validate_trace_producer_json( + trace_schema_input: &[u8], + trace_input: &[u8], + transcript_schema_input: &[u8], + transcript_input: &[u8], +) -> ValidationReport { + let trace_structure = validate_instance_json(trace_schema_input, trace_input); + if !trace_structure.conformant { + return ValidationReport { + kind: "trace-producer".to_owned(), + conformant: false, + diagnostics: trace_structure.diagnostics, + }; + } + let transcript_structure = validate_instance_json(transcript_schema_input, transcript_input); + if !transcript_structure.conformant { + return ValidationReport { + kind: "trace-producer".to_owned(), + conformant: false, + diagnostics: transcript_structure.diagnostics, + }; + } + + let trace = match serde_json::from_slice::(trace_input) { + Ok(value) => value, + Err(error) => { + return ValidationReport::invalid_input( + "trace-producer", + "INSTANCE_JSON_INVALID", + error.to_string(), + ); + } + }; + let transcript = match serde_json::from_slice::(transcript_input) { + Ok(value) => value, + Err(error) => { + return ValidationReport::invalid_input( + "trace-producer", + "INSTANCE_JSON_INVALID", + error.to_string(), + ); + } + }; + let diagnostics = trace::validate_trace(&trace, Some(&transcript)); + ValidationReport { + kind: "trace-producer".to_owned(), + conformant: diagnostics.is_empty(), + diagnostics, + } +} + #[must_use] pub fn check_requirements(document: &RequirementsDocument) -> CheckReport { let mut diagnostics = Vec::new(); diff --git a/conformance/runner/src/main.rs b/conformance/runner/src/main.rs index dce277c..7e531ea 100644 --- a/conformance/runner/src/main.rs +++ b/conformance/runner/src/main.rs @@ -2,14 +2,18 @@ use std::env; use std::fs; +use std::io::{self, Write as _}; use std::process::ExitCode; use promptsyntax_conformance::{ - CheckReport, ValidationReport, check_requirements_json, check_schema_json, run_suite, - validate_instance_json, + CheckReport, CoreDifferentialReport, ValidationReport, check_requirements_json, + check_schema_json, compare_core_adapter_streams, compare_core_adapters_json, run_suite, + run_trace_producer_adapter, validate_instance_json, validate_trace_producer_json, + write_generated_core_cases, }; use serde::Serialize; +#[allow(clippy::too_many_lines)] fn main() -> ExitCode { let mut args = env::args_os(); let _program = args.next(); @@ -17,6 +21,36 @@ fn main() -> ExitCode { return usage(); }; match command.to_str() { + Some("generate-core-differential") => { + let Some(case_count) = args + .next() + .and_then(|value| value.into_string().ok()) + .and_then(|value| value.parse::().ok()) + else { + return usage(); + }; + let Some(seed) = args + .next() + .and_then(|value| value.into_string().ok()) + .and_then(|value| value.parse::().ok()) + else { + return usage(); + }; + if args.next().is_some() || !(1..=1_000_000).contains(&case_count) { + return usage(); + } + let stdout = io::stdout(); + let mut output = stdout.lock(); + if let Err(error) = write_generated_core_cases(&mut output, case_count, seed) { + eprintln!("failed to generate Core differential inputs: {error}"); + return ExitCode::from(2); + } + if let Err(error) = writeln!(output) { + eprintln!("failed to finish Core differential output: {error}"); + return ExitCode::from(2); + } + ExitCode::SUCCESS + } Some("check-requirements") => { let Some(path) = args.next() else { return usage(); @@ -82,6 +116,74 @@ fn main() -> ExitCode { let report = validate_instance_json(&schema, &instance); emit_report(&report, report.conformant) } + Some("validate-trace-producer") => { + let Some(trace_schema_path) = args.next() else { + return usage(); + }; + let Some(trace_path) = args.next() else { + return usage(); + }; + let Some(transcript_schema_path) = args.next() else { + return usage(); + }; + let Some(transcript_path) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let trace_schema = match fs::read(&trace_schema_path) { + Ok(input) => input, + Err(error) => { + let report = ValidationReport::invalid_input( + "trace-producer", + "SCHEMA_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let trace = match fs::read(&trace_path) { + Ok(input) => input, + Err(error) => { + let report = ValidationReport::invalid_input( + "trace-producer", + "INSTANCE_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let transcript_schema = match fs::read(&transcript_schema_path) { + Ok(input) => input, + Err(error) => { + let report = ValidationReport::invalid_input( + "trace-producer", + "SCHEMA_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let transcript = match fs::read(&transcript_path) { + Ok(input) => input, + Err(error) => { + let report = ValidationReport::invalid_input( + "trace-producer", + "INSTANCE_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let report = validate_trace_producer_json( + &trace_schema, + &trace, + &transcript_schema, + &transcript, + ); + emit_report(&report, report.conformant) + } Some("run-suite") => { let Some(root) = args.next() else { return usage(); @@ -92,6 +194,119 @@ fn main() -> ExitCode { let report = run_suite(std::path::Path::new(&root)); emit_report(&report, report.conformant) } + Some("run-trace-producer-adapter") => { + let Some(root) = args.next() else { + return usage(); + }; + let Some(implementation_id) = args.next().and_then(|value| value.into_string().ok()) + else { + return usage(); + }; + let Some(implementation_version) = + args.next().and_then(|value| value.into_string().ok()) + else { + return usage(); + }; + let Some(implementation_commit) = + args.next().and_then(|value| value.into_string().ok()) + else { + return usage(); + }; + let command = args.collect::>(); + let report = match run_trace_producer_adapter( + std::path::Path::new(&root), + implementation_id, + implementation_version, + implementation_commit, + &command, + ) { + Ok(report) => report, + Err(error) => { + eprintln!("failed to run Trace producer adapter: {error}"); + return ExitCode::from(2); + } + }; + let conformant = report.conformant(); + emit_report(&report, conformant) + } + Some("compare-core-adapters") => { + let Some(cases_path) = args.next() else { + return usage(); + }; + let Some(left_path) = args.next() else { + return usage(); + }; + let Some(right_path) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let cases = match fs::read(&cases_path) { + Ok(input) => input, + Err(error) => { + let report = CoreDifferentialReport::invalid_input( + "CORE_CASES_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let left = match fs::read(&left_path) { + Ok(input) => input, + Err(error) => { + let report = CoreDifferentialReport::invalid_input( + "LEFT_ADAPTER_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let right = match fs::read(&right_path) { + Ok(input) => input, + Err(error) => { + let report = CoreDifferentialReport::invalid_input( + "RIGHT_ADAPTER_READ_FAILED", + error.to_string(), + ); + return emit_report(&report, false); + } + }; + let report = compare_core_adapters_json(&cases, &left, &right); + emit_report(&report, report.compatible) + } + Some("compare-core-streams") => { + let Some(case_count) = args + .next() + .and_then(|value| value.into_string().ok()) + .and_then(|value| value.parse::().ok()) + else { + return usage(); + }; + let Some(seed) = args + .next() + .and_then(|value| value.into_string().ok()) + .and_then(|value| value.parse::().ok()) + else { + return usage(); + }; + let Some(left_path) = args.next() else { + return usage(); + }; + let Some(right_path) = args.next() else { + return usage(); + }; + if args.next().is_some() || !(1..=1_000_000).contains(&case_count) { + return usage(); + } + let report = compare_core_adapter_streams( + case_count, + seed, + std::path::Path::new(&left_path), + std::path::Path::new(&right_path), + ); + emit_report(&report, report.compatible) + } _ => usage(), } } @@ -114,7 +329,7 @@ fn emit_report(report: &impl Serialize, conformant: bool) -> ExitCode { fn usage() -> ExitCode { eprintln!( - "usage:\n ps-conformance check-requirements \n ps-conformance check-schema \n ps-conformance validate-instance \n ps-conformance run-suite " + "usage:\n ps-conformance generate-core-differential \n ps-conformance check-requirements \n ps-conformance check-schema \n ps-conformance validate-instance \n ps-conformance validate-trace-producer \n ps-conformance run-suite \n ps-conformance run-trace-producer-adapter [args...]\n ps-conformance compare-core-adapters \n ps-conformance compare-core-streams " ); ExitCode::from(2) } diff --git a/conformance/runner/src/producer.rs b/conformance/runner/src/producer.rs new file mode 100644 index 0000000..8c72c8d --- /dev/null +++ b/conformance/runner/src/producer.rs @@ -0,0 +1,305 @@ +use std::ffi::OsString; +use std::fs; +use std::path::{Component, Path}; +use std::process::Command; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::validate_trace_producer_json; + +const FORMAT_VERSION: &str = "0.1-draft"; +const CASES_PATH: &str = "conformance/cases/trace-producer.json"; +const CASES_SCHEMA_PATH: &str = "conformance/trace-producer-cases.schema.json"; +const INPUT_SCHEMA_PATH: &str = "conformance/trace-producer-input.schema.json"; +const TRACE_SCHEMA_PATH: &str = "schemas/prompt-trace-0.3-draft.schema.json"; +const TRANSCRIPT_SCHEMA_PATH: &str = "conformance/transcript.schema.json"; + +#[derive(Debug, Serialize)] +pub struct TraceProducerAdapterReport { + pub format_version: String, + pub target: String, + pub implementation: Implementation, + pub results: Vec, +} + +impl TraceProducerAdapterReport { + #[must_use] + pub fn conformant(&self) -> bool { + self.results.iter().all(|result| result.conformant) + } +} + +#[derive(Debug, Serialize)] +pub struct Implementation { + pub id: String, + pub version: String, + pub commit: String, +} + +#[derive(Debug, Serialize)] +pub struct ProducerCaseResult { + pub case_id: String, + pub conformant: bool, + pub diagnostics: Vec, + pub output: Value, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ProducerCases { + format_version: String, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ProducerCase { + id: String, + input: String, + transcript: Option, + expected: Expected, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "outcome", rename_all = "lowercase")] +enum Expected { + Trace { + conformant: bool, + diagnostics: Vec, + }, + Error { + code: String, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ErrorEnvelope { + format_version: String, + error: ProducerError, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct ProducerError { + code: String, + pointer: String, + message: String, +} + +/// Run the canonical producer corpus against an external implementation. +/// +/// # Errors +/// +/// Returns an error when suite artifacts cannot be loaded, implementation +/// metadata is invalid, or the external producer process cannot be started. +pub fn run_trace_producer_adapter( + root: &Path, + implementation_id: String, + implementation_version: String, + implementation_commit: String, + command: &[OsString], +) -> Result { + if command.is_empty() { + return Err("Trace producer command is empty".to_owned()); + } + if implementation_id.is_empty() || implementation_version.is_empty() { + return Err("implementation id and version must not be empty".to_owned()); + } + if implementation_commit.len() != 40 + || !implementation_commit + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err("implementation commit must be 40 hexadecimal characters".to_owned()); + } + + let cases_bytes = read(root, CASES_PATH)?; + let cases_schema = read(root, CASES_SCHEMA_PATH)?; + let cases_validation = crate::validate_instance_json(&cases_schema, &cases_bytes); + if !cases_validation.conformant { + return Err("Trace producer cases do not satisfy their schema".to_owned()); + } + let cases: ProducerCases = serde_json::from_slice(&cases_bytes) + .map_err(|error| format!("invalid Trace producer cases: {error}"))?; + if cases.format_version != FORMAT_VERSION { + return Err(format!( + "Trace producer case format must be {FORMAT_VERSION}" + )); + } + let input_schema = read(root, INPUT_SCHEMA_PATH)?; + let trace_schema = read(root, TRACE_SCHEMA_PATH)?; + let transcript_schema = read(root, TRANSCRIPT_SCHEMA_PATH)?; + let mut results = Vec::with_capacity(cases.cases.len()); + + for case in cases.cases { + results.push(run_case( + root, + case, + &input_schema, + &trace_schema, + &transcript_schema, + command, + )); + } + + Ok(TraceProducerAdapterReport { + format_version: FORMAT_VERSION.to_owned(), + target: "trace-producer".to_owned(), + implementation: Implementation { + id: implementation_id, + version: implementation_version, + commit: implementation_commit, + }, + results, + }) +} + +#[allow(clippy::too_many_lines)] +fn run_case( + root: &Path, + case: ProducerCase, + input_schema: &[u8], + trace_schema: &[u8], + transcript_schema: &[u8], + command: &[OsString], +) -> ProducerCaseResult { + let input = match read(root, &case.input) { + Ok(input) => input, + Err(error) => return failed_case(case.id, "TRACE_PRODUCER_INPUT_READ_FAILED", error), + }; + let input_validation = crate::validate_instance_json(input_schema, &input); + if !input_validation.conformant { + return ProducerCaseResult { + case_id: case.id, + conformant: false, + diagnostics: input_validation + .diagnostics + .into_iter() + .map(|diagnostic| diagnostic.code) + .collect(), + output: json!({ "error": "producer input does not satisfy its schema" }), + }; + } + + let input_path = root.join(&case.input); + let output = match Command::new(&command[0]) + .args(&command[1..]) + .arg(&input_path) + .output() + { + Ok(output) => output, + Err(error) => { + return failed_case(case.id, "TRACE_PRODUCER_PROCESS_FAILED", error.to_string()); + } + }; + let value = match serde_json::from_slice::(&output.stdout) { + Ok(value) => value, + Err(error) => { + return ProducerCaseResult { + case_id: case.id, + conformant: false, + diagnostics: vec!["TRACE_PRODUCER_OUTPUT_INVALID".to_owned()], + output: json!({ + "status": output.status.code(), + "stdout": String::from_utf8_lossy(&output.stdout), + "stderr": String::from_utf8_lossy(&output.stderr), + "parse_error": error.to_string(), + }), + }; + } + }; + + match case.expected { + Expected::Trace { + conformant, + mut diagnostics, + } => { + if !output.status.success() { + return ProducerCaseResult { + case_id: case.id, + conformant: false, + diagnostics: vec!["TRACE_PRODUCER_PROCESS_FAILED".to_owned()], + output: value, + }; + } + let Some(transcript_path) = case.transcript else { + return failed_case( + case.id, + "TRACE_PRODUCER_TRANSCRIPT_REQUIRED", + "trace case has no independent transcript", + ); + }; + let transcript = match read(root, &transcript_path) { + Ok(transcript) => transcript, + Err(error) => { + return failed_case(case.id, "TRACE_PRODUCER_TRANSCRIPT_READ_FAILED", error); + } + }; + let trace_bytes = + serde_json::to_vec(&value).expect("serializing parsed producer output cannot fail"); + let validation = validate_trace_producer_json( + trace_schema, + &trace_bytes, + transcript_schema, + &transcript, + ); + let mut actual = validation + .diagnostics + .into_iter() + .map(|diagnostic| diagnostic.code) + .collect::>(); + actual.sort(); + actual.dedup(); + diagnostics.sort(); + diagnostics.dedup(); + ProducerCaseResult { + case_id: case.id, + conformant: validation.conformant == conformant && actual == diagnostics, + diagnostics: actual, + output: value, + } + } + Expected::Error { code } => { + let envelope = serde_json::from_value::(value.clone()); + let (matches, diagnostics) = match envelope { + Ok(envelope) => ( + output.status.code() == Some(1) + && envelope.format_version == FORMAT_VERSION + && envelope.error.code == code, + vec![envelope.error.code], + ), + Err(_) => (false, vec!["TRACE_PRODUCER_OUTPUT_INVALID".to_owned()]), + }; + ProducerCaseResult { + case_id: case.id, + conformant: matches, + diagnostics, + output: value, + } + } + } +} + +fn failed_case(case_id: String, code: &str, message: impl Into) -> ProducerCaseResult { + let message = message.into(); + ProducerCaseResult { + case_id, + conformant: false, + diagnostics: vec![code.to_owned()], + output: json!({ "error": message }), + } +} + +fn read(root: &Path, relative: &str) -> Result, String> { + let path = Path::new(relative); + if path.is_absolute() + || path + .components() + .any(|component| matches!(component, Component::ParentDir | Component::RootDir)) + { + return Err(format!("unsafe repository-relative path: {relative}")); + } + fs::read(root.join(path)).map_err(|error| format!("failed to read {relative}: {error}")) +} diff --git a/conformance/runner/src/suite.rs b/conformance/runner/src/suite.rs index fe6a8ff..812e75b 100644 --- a/conformance/runner/src/suite.rs +++ b/conformance/runner/src/suite.rs @@ -106,15 +106,25 @@ struct ManifestFiles { trace_schema: String, family_schema: String, transcript_schema: String, + trace_producer_input_schema: String, + trace_producer_cases: String, + trace_producer_cases_schema: String, profile: String, profile_schema: String, report_schema: String, core_cases: String, core_cases_schema: String, + core_differential_input_schema: String, + core_adapter_result_schema: String, + core_differential_report_schema: String, adapter_result_schema: String, implementations_lock: String, implementations_schema: String, candidate_report: String, + core_differential_report: String, + core_generated_differential_report: String, + trace_producer_rs_report: String, + trace_producer_ts_report: String, } #[derive(Debug, Deserialize)] @@ -179,6 +189,20 @@ struct PatchOperation { value: Option, } +#[derive(Debug, Deserialize)] +struct ProducerCaseCorpus { + format_version: String, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct ProducerCorpusCase { + id: String, + input: String, + transcript: Option, + expected: Value, +} + #[must_use] pub fn run_suite(root: &Path) -> SuiteReport { match run_suite_inner(root) { @@ -261,10 +285,20 @@ fn run_suite_inner(root: &Path) -> Result { let trace_schema = checked_schema(root, &manifest.files.trace_schema)?; let family_schema = checked_schema(root, &manifest.files.family_schema)?; let transcript_schema = checked_schema(root, &manifest.files.transcript_schema)?; + let trace_producer_input_schema = + checked_schema(root, &manifest.files.trace_producer_input_schema)?; + let trace_producer_cases_schema = + checked_schema(root, &manifest.files.trace_producer_cases_schema)?; let profile_schema = checked_schema(root, &manifest.files.profile_schema)?; let report_schema = checked_schema(root, &manifest.files.report_schema)?; let core_cases_schema = checked_schema(root, &manifest.files.core_cases_schema)?; - let _adapter_result_schema = checked_schema(root, &manifest.files.adapter_result_schema)?; + let _core_differential_input_schema = + checked_schema(root, &manifest.files.core_differential_input_schema)?; + let _core_adapter_result_schema = + checked_schema(root, &manifest.files.core_adapter_result_schema)?; + let core_differential_report_schema = + checked_schema(root, &manifest.files.core_differential_report_schema)?; + let adapter_result_schema = checked_schema(root, &manifest.files.adapter_result_schema)?; let core_cases = read_json(root, &manifest.files.core_cases)?; append_validation( @@ -279,12 +313,43 @@ fn run_suite_inner(root: &Path) -> Result { validate_value(&implementations_schema, &implementations_lock), &manifest.files.implementations_lock, ); + validate_producer_corpus( + root, + &manifest.files.trace_producer_cases, + &trace_producer_cases_schema, + &trace_producer_input_schema, + &transcript_schema, + &diagnostic_codes, + &mut suite_diagnostics, + )?; let candidate_report = read_json(root, &manifest.files.candidate_report)?; append_validation( &mut suite_diagnostics, validate_value(&report_schema, &candidate_report), &manifest.files.candidate_report, ); + for report_path in [ + &manifest.files.core_differential_report, + &manifest.files.core_generated_differential_report, + ] { + let report = read_json(root, report_path)?; + append_validation( + &mut suite_diagnostics, + validate_value(&core_differential_report_schema, &report), + report_path, + ); + } + for report_path in [ + &manifest.files.trace_producer_rs_report, + &manifest.files.trace_producer_ts_report, + ] { + let report = read_json(root, report_path)?; + append_validation( + &mut suite_diagnostics, + validate_value(&adapter_result_schema, &report), + report_path, + ); + } let profile_value = read_json(root, &manifest.files.profile)?; append_validation( @@ -585,6 +650,77 @@ fn load_transcript( } } +#[allow(clippy::too_many_arguments)] +fn validate_producer_corpus( + root: &Path, + cases_path: &str, + cases_schema: &Value, + input_schema: &Value, + transcript_schema: &Value, + diagnostic_codes: &HashSet, + diagnostics: &mut Vec, +) -> Result<(), Diagnostic> { + let cases_value = read_json(root, cases_path)?; + let cases_validation = validate_value(cases_schema, &cases_value); + append_validation(diagnostics, cases_validation.clone(), cases_path); + if !cases_validation.conformant { + return Ok(()); + } + let corpus: ProducerCaseCorpus = deserialize_value(cases_value, cases_path)?; + if corpus.format_version != FORMAT_VERSION { + push( + diagnostics, + "SUITE_FORMAT_UNSUPPORTED", + cases_path, + format!("Trace producer case format must be {FORMAT_VERSION}"), + ); + } + + let mut case_ids = HashSet::new(); + for case in corpus.cases { + if !case_ids.insert(case.id.clone()) { + push( + diagnostics, + "SUITE_CASE_ID_DUPLICATE", + cases_path, + format!("Trace producer case id {} occurs more than once", case.id), + ); + } + match read_json(root, &case.input) { + Ok(input) => append_validation( + diagnostics, + validate_value(input_schema, &input), + &case.input, + ), + Err(error) => diagnostics.push(error), + } + if let Some(transcript_path) = &case.transcript { + match read_json(root, transcript_path) { + Ok(transcript) => append_validation( + diagnostics, + validate_value(transcript_schema, &transcript), + transcript_path, + ), + Err(error) => diagnostics.push(error), + } + } + if let Some(code) = case.expected.get("code").and_then(Value::as_str) { + if !diagnostic_codes.contains(code) { + push( + diagnostics, + "SUITE_DIAGNOSTIC_UNREGISTERED", + cases_path, + format!( + "Trace producer case {} expects unregistered code {code}", + case.id + ), + ); + } + } + } + Ok(()) +} + fn validate_family_metadata( entry: &ManifestFamily, family: &CaseFamily, diff --git a/conformance/trace-producer-cases.schema.json b/conformance/trace-producer-cases.schema.json new file mode 100644 index 0000000..9a08d2b --- /dev/null +++ b/conformance/trace-producer-cases.schema.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/trace-producer-cases/0.1-draft", + "title": "PromptSyntax Trace producer adapter cases", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "cases"], + "properties": { + "format_version": { "const": "0.1-draft" }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "input", "expected"], + "properties": { + "id": { "type": "string", "pattern": "^producer-[a-z0-9]+(?:-[a-z0-9]+)*$" }, + "input": { "type": "string", "minLength": 1 }, + "transcript": { "type": "string", "minLength": 1 }, + "expected": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["outcome", "conformant", "diagnostics"], + "properties": { + "outcome": { "const": "trace" }, + "conformant": { "const": true }, + "diagnostics": { + "type": "array", + "maxItems": 0 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["outcome", "code"], + "properties": { + "outcome": { "const": "error" }, + "code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" } + } + } + ] + } + }, + "allOf": [ + { + "if": { + "required": ["expected"], + "properties": { + "expected": { + "required": ["outcome"], + "properties": { "outcome": { "const": "trace" } } + } + } + }, + "then": { "required": ["transcript"] }, + "else": { "not": { "required": ["transcript"] } } + } + ] + } + } + } +} diff --git a/conformance/trace-producer-input.schema.json b/conformance/trace-producer-input.schema.json new file mode 100644 index 0000000..c7aa3d8 --- /dev/null +++ b/conformance/trace-producer-input.schema.json @@ -0,0 +1,375 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/trace-producer-input/0.1-draft", + "title": "PromptSyntax deterministic Trace producer input", + "description": "Execution facts supplied to a Trace producer. This document contains no expected Trace output.", + "type": "object", + "additionalProperties": false, + "required": [ + "format_version", + "turn", + "tier", + "mode", + "content_mode", + "venue", + "coverage", + "inferences", + "boundaries" + ], + "properties": { + "format_version": { "const": "0.1-draft" }, + "turn": { "type": "string", "minLength": 1 }, + "tier": { "const": "user" }, + "mode": { "const": "executed" }, + "content_mode": { "const": "tiered" }, + "venue": { "$ref": "#/$defs/venue" }, + "coverage": { "$ref": "#/$defs/coverage" }, + "inferences": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/inference" } + }, + "boundaries": { + "type": "array", + "items": { "$ref": "#/$defs/boundary" } + } + }, + "$defs": { + "venue": { + "type": "object", + "additionalProperties": false, + "required": ["id", "capability_document"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "capability_document": { "type": "string", "format": "uri" }, + "vendor_profile": { "type": "string", "minLength": 1 } + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["routing", "assembly", "inline_threshold_bytes"], + "properties": { + "routing": { "const": "complete" }, + "assembly": { "const": "user-initiated-only" }, + "inline_threshold_bytes": { "type": "integer", "minimum": 4096 }, + "profile": { "type": "string", "minLength": 1 } + } + }, + "inference": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "compiled_request_utf8", "routing"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "const": "user-initiated" }, + "compiled_request_utf8": { "type": "string" }, + "routing": { "$ref": "#/$defs/routing" } + } + }, + "routing": { + "type": "object", + "additionalProperties": false, + "required": ["requested", "policy", "route", "attempts", "non_entity_fill"], + "properties": { + "requested": { "type": "string", "minLength": 1 }, + "policy": { "enum": ["strict", "partial", "best-effort"] }, + "route": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/route_step" } + }, + "attempts": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/attempt" } + }, + "non_entity_fill": { + "type": "array", + "items": { + "oneOf": [ + { "$ref": "#/$defs/parameter_fill" }, + { "$ref": "#/$defs/budget_fill" }, + { "$ref": "#/$defs/action_fill" } + ] + } + }, + "refusal": { "$ref": "#/$defs/refusal_fact" } + } + }, + "route_step": { + "type": "object", + "additionalProperties": false, + "required": ["ref", "canonical", "rule", "ambiguity_surfaced"], + "properties": { + "ref": { "type": "string", "minLength": 1 }, + "canonical": { "$ref": "#/$defs/canonical_entity" }, + "rule": { "type": "string", "minLength": 1 }, + "ambiguity_surfaced": { "type": "boolean" } + } + }, + "attempt": { + "type": "object", + "additionalProperties": false, + "required": ["ref", "outcome"], + "properties": { + "ref": { "type": "string", "minLength": 1 }, + "bound": { "$ref": "#/$defs/canonical_entity" }, + "outcome": { "enum": ["filled", "failed", "blocked"] }, + "reasons": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/outcome_reason" } + }, + "measured": { "$ref": "#/$defs/measured" } + }, + "allOf": [ + { + "if": { + "required": ["outcome"], + "properties": { "outcome": { "const": "filled" } } + }, + "then": { "required": ["bound", "measured"] }, + "else": { + "required": ["reasons"], + "not": { "required": ["bound"] } + } + } + ] + }, + "refusal_fact": { + "type": "object", + "additionalProperties": false, + "required": ["reason", "authority", "recourse"], + "properties": { + "reason": { "$ref": "#/$defs/outcome_reason" }, + "authority": { "$ref": "#/$defs/authority" }, + "triggering_step": { "type": "string", "minLength": 1 }, + "recourse": { "$ref": "#/$defs/recourse" }, + "policy_decision_id": { "type": "string", "minLength": 1 }, + "policy_bundle": { "type": "string", "minLength": 1 }, + "policy_engine": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "if": { + "required": ["reason"], + "properties": { "reason": { "const": "SAFETY_BLOCKED" } } + }, + "then": { "required": ["triggering_step"] } + } + ] + }, + "boundary": { + "type": "object", + "additionalProperties": false, + "required": ["id", "tool", "outcome", "sent", "fill", "measured"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "tool": { "type": "string", "minLength": 1 }, + "outcome": { "enum": ["succeeded", "failed", "blocked", "cancelled"] }, + "sent": { "$ref": "#/$defs/content" }, + "received": { "$ref": "#/$defs/content" }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "fill": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { "$ref": "#/$defs/parameter_fill" }, + { "$ref": "#/$defs/budget_fill" }, + { "$ref": "#/$defs/action_fill" } + ] + } + }, + "internal": { "const": "outside-trace-perimeter" }, + "measured": { "$ref": "#/$defs/measured" } + }, + "allOf": [ + { + "if": { + "required": ["outcome"], + "properties": { "outcome": { "const": "succeeded" } } + }, + "then": { "required": ["received"] }, + "else": { "required": ["reason"] } + } + ] + }, + "parameter_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "name", "requested", "status", "policy"], + "properties": { + "kind": { "const": "parameter" }, + "name": { "type": "string", "minLength": 1 }, + "requested": true, + "normalized": true, + "applied": true, + "status": { + "enum": ["applied", "translated", "clamped", "unsupported", "ignored", "refused"] + }, + "policy": { "enum": ["strict", "partial", "best-effort"] }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "mapping_rule": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "enum": ["applied", "translated", "clamped"] } } + }, + "then": { "required": ["applied"] }, + "else": { "required": ["reason"] } + }, + { + "if": { + "required": ["status"], + "properties": { "status": { "enum": ["translated", "clamped"] } } + }, + "then": { "required": ["normalized", "mapping_rule"] } + } + ] + }, + "budget_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "name", "requested", "status"], + "properties": { + "kind": { "const": "budget" }, + "name": { "type": "string", "minLength": 1 }, + "requested": true, + "applied": true, + "status": { "enum": ["applied", "exceeded", "unsupported", "refused"] }, + "reason": { "$ref": "#/$defs/outcome_reason" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "const": "applied" } } + }, + "then": { "required": ["applied"] }, + "else": { "required": ["reason"] } + } + ] + }, + "action_fill": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "requested", "status"], + "properties": { + "kind": { "const": "action" }, + "requested": { "type": "string", "minLength": 1 }, + "applied": { "type": "string", "minLength": 1 }, + "status": { "enum": ["applied", "refused"] }, + "reason": { "$ref": "#/$defs/outcome_reason" }, + "deciding_authority": { "$ref": "#/$defs/authority" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { "status": { "const": "applied" } } + }, + "then": { "required": ["applied"] }, + "else": { "required": ["reason", "deciding_authority"] } + } + ] + }, + "content": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["state", "text"], + "properties": { + "state": { "const": "inline" }, + "text": { "type": "string" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["state", "sha256", "length", "handle"], + "properties": { + "state": { "const": "external" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "length": { "type": "integer", "minimum": 0 }, + "media": { "type": "string", "minLength": 1 }, + "handle": { "type": "string", "format": "uri" } + } + } + ] + }, + "measured": { + "type": "object", + "additionalProperties": false, + "properties": { + "cost_usd": { "type": "number", "minimum": 0 }, + "wall_time_ms": { "type": "integer", "minimum": 0 }, + "first_token_ms": { "type": "integer", "minimum": 0 }, + "cancelled": { "type": "boolean" } + } + }, + "recourse": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["available", "kind"], + "properties": { + "available": { "const": true }, + "kind": { "enum": ["appeal", "verification", "feedback", "retry"] }, + "uri": { "type": "string", "format": "uri" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["available", "reason"], + "properties": { + "available": { "const": false }, + "reason": { "type": "string", "minLength": 1 } + } + } + ] + }, + "canonical_entity": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*:[^@]+/[^@]+@[^@]+$" + }, + "authority": { + "enum": [ + "law-regulation", + "platform", + "venue-operations", + "organization", + "application", + "user", + "prompt" + ] + }, + "outcome_reason": { + "enum": [ + "ENTITY_NOT_FOUND", + "ENTITY_AMBIGUOUS", + "ENTITY_UNAVAILABLE", + "ENTITY_SUBSTITUTED", + "ENTITY_UNAUTHORIZED", + "PARAM_UNSUPPORTED", + "PARAM_CLAMPED", + "PARAM_IGNORED", + "BUDGET_PREFLIGHT_REJECTED", + "BUDGET_EXCEEDED", + "TIMEOUT_NOT_CANCELLED", + "DATA_POLICY_BLOCKED", + "SAFETY_BLOCKED", + "SPAN_EXECUTION_UNSUPPORTED", + "VERSION_UNSUPPORTED", + "FALLBACK_EXHAUSTED" + ] + } + } +} From 4ba418ff419f9c0006b6433347f6cafec7fb21eb Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 03:09:43 +0700 Subject: [PATCH 5/5] ci: install implementation lint components --- .github/workflows/conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 2dd1c83..8c9c9f9 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -79,6 +79,7 @@ jobs: uses: dtolnay/rust-toolchain@stable with: toolchain: 1.97.0 + components: rustfmt, clippy - name: Install Bun uses: oven-sh/setup-bun@v2