From 5ae8c139c4e2d66d2e24c435d61c9abe0cf7c586 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 06:17:46 +0700 Subject: [PATCH] feat: add adversarial Trace benchmark --- .github/workflows/conformance.yml | 13 +- conformance/README.md | 40 +- conformance/adversarial-benchmark.schema.json | 142 +++ conformance/adversarial-report.schema.json | 200 ++++ .../trace-adversarial-0.1.0-rc.1.json | 241 +++++ conformance/core-adapter-result.schema.json | 8 +- conformance/diagnostic-registry.json | 96 ++ conformance/diagnostic-registry.schema.json | 11 +- .../families/trace-boundary-tamper.json | 4 + conformance/families/trace-outcomes.json | 17 +- conformance/families/trace-structure.json | 65 +- conformance/implementations.lock.json | 40 +- conformance/manifest.json | 4 + conformance/manifest.schema.json | 8 + conformance/releases/0.1.0-rc.1.md | 10 +- conformance/reports/0.1.0-rc.1.json | 15 +- .../reports/trace-adversarial-0.1.0-rc.1.json | 562 ++++++++++++ conformance/runner/README.md | 9 + conformance/runner/src/adversarial.rs | 865 ++++++++++++++++++ conformance/runner/src/lib.rs | 2 + conformance/runner/src/main.rs | 18 +- conformance/runner/src/suite.rs | 82 +- 22 files changed, 2413 insertions(+), 39 deletions(-) create mode 100644 conformance/adversarial-benchmark.schema.json create mode 100644 conformance/adversarial-report.schema.json create mode 100644 conformance/benchmarks/trace-adversarial-0.1.0-rc.1.json create mode 100644 conformance/reports/trace-adversarial-0.1.0-rc.1.json create mode 100644 conformance/runner/src/adversarial.rs diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 8c9c9f9..a2af234 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -26,8 +26,12 @@ permissions: jobs: suite: - name: Trace suite - runs-on: ubuntu-latest + name: Trace suite (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: Checkout standard and corpus uses: actions/checkout@v4 @@ -52,6 +56,11 @@ jobs: cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-suite . > /tmp/promptsyntax-conformance-report.json cmp conformance/reports/0.1.0-rc.1.json /tmp/promptsyntax-conformance-report.json + - name: Verify adversarial benchmark snapshot + run: | + cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-adversarial-benchmark . > /tmp/promptsyntax-trace-adversarial.json + cmp conformance/reports/trace-adversarial-0.1.0-rc.1.json /tmp/promptsyntax-trace-adversarial.json + cross-language-core: name: Cross-language Core and Trace adapters runs-on: ubuntu-latest diff --git a/conformance/README.md b/conformance/README.md index bebf2e0..808a36f 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -7,7 +7,7 @@ are the oracle. Implementations consume the corpus but do not define its expecte ## Current status Version `0.1.0-rc.1` is a candidate suite against the specification draft at commit -`7456634817f4eb68c8909a58b476505e6fd7c063`. It executes 34 document and producer cases +`7456634817f4eb68c8909a58b476505e6fd7c063`. It executes 35 document and producer cases across four Trace families and covers all 22 requirements in its named candidate profile. It also owns 101 Core parser vectors and a full normalized-output differential protocol for the Rust and TypeScript implementations. A deterministic grammar generator adds a @@ -32,6 +32,34 @@ The normative runner does not require Python, Node, Bun, a provider SDK, a netwo connection, or model credentials. The TypeScript implementation may be invoked separately with Bun when producing cross-language evidence. +## Adversarial mutation benchmark + +The candidate suite also defines a scored adversarial benchmark over specification-owned +Trace cases. Each mutation is paired with a conformant control. A mutation earns credit +only when the control still passes and the mutant fails with the exact intended diagnostic +set; merely rejecting the document is insufficient. The `0.1.0-rc.1` snapshot accepts all +10 unique controls and exactly detects all 18 mutations across assembly and provenance, +content integrity, routing and resolution, event and boundary integrity, and independent +producer replay. The runner also enforces the declared selection rule: every negative case +in the candidate families whose expected diagnostics are all Trace-specific must appear +exactly once in the score. This makes the current result exhaustive over that bounded +semantic-negative corpus rather than a hand-selected subset. The runner materializes each +control and mutant, computes their observed JSON-pointer differences, and rejects a pair if +any difference escapes its declared controlled mutation paths. + +Run and reproduce the frozen report: + +```bash +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + run-adversarial-benchmark . +``` + +This benchmark measures detection of the bounded semantic mutations in its manifest. It +does not establish cryptographic receipt authenticity, truthfulness without independent +producer facts, resistance to resource exhaustion, transport-decoder security, or behavior +for semantics outside the candidate profile. Those exclusions are machine-readable in the +benchmark document and must remain explicit in paper claims. + Run the repository-owned checker from the repository root: ```bash @@ -40,6 +68,9 @@ cargo run --manifest-path conformance/runner/Cargo.toml -- \ cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + run-adversarial-benchmark . + cargo run --manifest-path conformance/runner/Cargo.toml -- \ compare-core-adapters conformance/cases/core-parser.json \ /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json @@ -58,16 +89,23 @@ cargo run --manifest-path conformance/runner/Cargo.toml -- \ The command emits the deterministic report stored in `conformance/reports/0.1.0-rc.1.json`. CI regenerates it and requires a byte-for-byte match. +The adversarial command likewise regenerates +`conformance/reports/trace-adversarial-0.1.0-rc.1.json` byte-for-byte. +The suite job performs both snapshot checks on Linux and macOS so diagnostic and report +stability is tested across operating systems and differing checkout paths. ## Layout - `profiles/` freezes the named scope and candidate semantic decisions. +- `benchmarks/` defines scored threat classes and paired semantic mutations. - `families/` contains language-independent cases and expected diagnostics. - `fixtures/` contains reusable Trace and deterministic transcript inputs. - `cases/core-parser.json` is the single 101-case Core fixture consumed by both adapters. - `core-differential-input.schema.json` freezes the generator-to-adapter input shape. - `core-adapter-result.schema.json` freezes the normalized cross-language output shape. - `core-differential-report.schema.json` freezes the independent comparison report shape. +- `adversarial-benchmark.schema.json` and `adversarial-report.schema.json` freeze the + benchmark and result contracts. - `cases/trace-producer.json` lists six valid and four rejection cases for real producers. - `trace-producer-input.schema.json` defines deterministic facts without expected output. - `trace-producer-cases.schema.json` freezes the external producer adapter protocol. diff --git a/conformance/adversarial-benchmark.schema.json b/conformance/adversarial-benchmark.schema.json new file mode 100644 index 0000000..dcba41c --- /dev/null +++ b/conformance/adversarial-benchmark.schema.json @@ -0,0 +1,142 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/adversarial-benchmark/0.1-draft", + "title": "PromptSyntax Trace adversarial mutation benchmark", + "type": "object", + "additionalProperties": false, + "required": [ + "format_version", + "id", + "version", + "status", + "suite", + "profile", + "evidence_unit", + "selection_rule", + "scoring", + "scope", + "threat_classes", + "mutations" + ], + "properties": { + "format_version": { "const": "0.1-draft" }, + "id": { "const": "trace-adversarial" }, + "version": { "type": "string", "minLength": 1 }, + "status": { "enum": ["candidate", "released"] }, + "suite": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version"], + "properties": { + "id": { "const": "promptsyntax-conformance" }, + "version": { "type": "string", "minLength": 1 } + } + }, + "profile": { "type": "string", "minLength": 1 }, + "evidence_unit": { "const": "paired-semantic-mutation" }, + "selection_rule": { "const": "all-negative-cases-with-trace-diagnostics" }, + "scoring": { + "type": "object", + "additionalProperties": false, + "required": ["control_acceptance", "mutation_detection", "aggregation"], + "properties": { + "control_acceptance": { "const": "baseline-must-pass-as-conformant" }, + "mutation_detection": { + "const": "mutant-must-fail-with-exact-diagnostic-set" + }, + "aggregation": { "const": "unweighted-case-count" } + } + }, + "scope": { + "type": "object", + "additionalProperties": false, + "required": ["claim", "attacker_capabilities", "excluded"], + "properties": { + "claim": { "type": "string", "minLength": 1 }, + "attacker_capabilities": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "excluded": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + } + } + }, + "threat_classes": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description"], + "properties": { + "id": { "$ref": "#/$defs/kebab_id" }, + "description": { "type": "string", "minLength": 1 } + } + } + }, + "mutations": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "threat_class", + "operator", + "baseline_case", + "mutant_case", + "controlled_paths", + "attack", + "intended_diagnostics" + ], + "properties": { + "id": { "type": "string", "pattern": "^A[0-9]{3}$" }, + "threat_class": { "$ref": "#/$defs/kebab_id" }, + "operator": { + "enum": [ + "content-corruption", + "cross-source-divergence", + "duplication", + "field-omission", + "field-replacement", + "metadata-corruption", + "policy-conflict" + ] + }, + "baseline_case": { "$ref": "#/$defs/kebab_id" }, + "mutant_case": { "$ref": "#/$defs/kebab_id" }, + "controlled_paths": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^/(trace|transcript)(/([^/~]|~0|~1)+)+$" + } + }, + "attack": { "type": "string", "minLength": 1 }, + "intended_diagnostics": { "$ref": "#/$defs/diagnostic_set" } + } + } + } + }, + "$defs": { + "kebab_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "diagnostic_set": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" } + } + } +} diff --git a/conformance/adversarial-report.schema.json b/conformance/adversarial-report.schema.json new file mode 100644 index 0000000..058b1e0 --- /dev/null +++ b/conformance/adversarial-report.schema.json @@ -0,0 +1,200 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promptsyntax.org/conformance/adversarial-report/0.1-draft", + "title": "PromptSyntax Trace adversarial benchmark report", + "type": "object", + "additionalProperties": false, + "required": [ + "format_version", + "benchmark", + "suite", + "profile", + "conformant", + "summary", + "rates", + "coverage", + "controls", + "threat_classes", + "mutations", + "diagnostics" + ], + "properties": { + "format_version": { "const": "0.1-draft" }, + "benchmark": { "$ref": "#/$defs/versioned_identity" }, + "suite": { "$ref": "#/$defs/versioned_identity" }, + "profile": { "$ref": "#/$defs/versioned_identity" }, + "conformant": { "type": "boolean" }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "threat_classes", + "unique_controls", + "controls_accepted", + "mutations", + "controlled_mutations", + "exactly_detected", + "rejected_with_unexpected_diagnostics", + "escaped", + "invalid_controls", + "missing_mutations" + ], + "properties": { + "threat_classes": { "$ref": "#/$defs/count" }, + "unique_controls": { "$ref": "#/$defs/count" }, + "controls_accepted": { "$ref": "#/$defs/count" }, + "mutations": { "$ref": "#/$defs/count" }, + "controlled_mutations": { "$ref": "#/$defs/count" }, + "exactly_detected": { "$ref": "#/$defs/count" }, + "rejected_with_unexpected_diagnostics": { "$ref": "#/$defs/count" }, + "escaped": { "$ref": "#/$defs/count" }, + "invalid_controls": { "$ref": "#/$defs/count" }, + "missing_mutations": { "$ref": "#/$defs/count" } + } + }, + "rates": { + "type": "object", + "additionalProperties": false, + "required": ["control_acceptance", "exact_mutation_detection"], + "properties": { + "control_acceptance": { "$ref": "#/$defs/ratio" }, + "exact_mutation_detection": { "$ref": "#/$defs/ratio" } + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["eligible_mutants", "scored_mutants", "missing", "extra"], + "properties": { + "eligible_mutants": { "$ref": "#/$defs/count" }, + "scored_mutants": { "$ref": "#/$defs/count" }, + "missing": { "$ref": "#/$defs/string_set" }, + "extra": { "$ref": "#/$defs/string_set" } + } + }, + "controls": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["case", "accepted", "actual_diagnostics"], + "properties": { + "case": { "type": "string", "minLength": 1 }, + "accepted": { "type": "boolean" }, + "actual_diagnostics": { "$ref": "#/$defs/string_set" } + } + } + }, + "threat_classes": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "description", + "mutations", + "exactly_detected", + "rejected_with_unexpected_diagnostics", + "escaped", + "invalid_controls", + "missing_mutations", + "exact_detection_rate" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "mutations": { "$ref": "#/$defs/count" }, + "exactly_detected": { "$ref": "#/$defs/count" }, + "rejected_with_unexpected_diagnostics": { "$ref": "#/$defs/count" }, + "escaped": { "$ref": "#/$defs/count" }, + "invalid_controls": { "$ref": "#/$defs/count" }, + "missing_mutations": { "$ref": "#/$defs/count" }, + "exact_detection_rate": { "$ref": "#/$defs/ratio" } + } + } + }, + "mutations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "threat_class", + "operator", + "baseline_case", + "mutant_case", + "controlled_change", + "controlled_paths", + "observed_paths", + "status", + "intended_diagnostics", + "actual_diagnostics" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "threat_class": { "type": "string", "minLength": 1 }, + "operator": { "type": "string", "minLength": 1 }, + "baseline_case": { "type": "string", "minLength": 1 }, + "mutant_case": { "type": "string", "minLength": 1 }, + "controlled_change": { "type": "boolean" }, + "controlled_paths": { "$ref": "#/$defs/string_set" }, + "observed_paths": { "$ref": "#/$defs/string_set" }, + "status": { + "enum": [ + "case-missing", + "escaped", + "exact-detection", + "invalid-control", + "unexpected-diagnostics" + ] + }, + "intended_diagnostics": { "$ref": "#/$defs/string_set" }, + "actual_diagnostics": { "$ref": "#/$defs/string_set" } + } + } + }, + "diagnostics": { + "type": "array", + "items": { "$ref": "#/$defs/diagnostic" } + } + }, + "$defs": { + "count": { "type": "integer", "minimum": 0 }, + "ratio": { + "type": "object", + "additionalProperties": false, + "required": ["numerator", "denominator"], + "properties": { + "numerator": { "$ref": "#/$defs/count" }, + "denominator": { "$ref": "#/$defs/count" } + } + }, + "versioned_identity": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "status"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "status": { "type": "string", "minLength": 1 } + } + }, + "string_set": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string" } + }, + "diagnostic": { + "type": "object", + "additionalProperties": false, + "required": ["code", "pointer", "message"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "pointer": { "type": "string" }, + "message": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/conformance/benchmarks/trace-adversarial-0.1.0-rc.1.json b/conformance/benchmarks/trace-adversarial-0.1.0-rc.1.json new file mode 100644 index 0000000..1c96705 --- /dev/null +++ b/conformance/benchmarks/trace-adversarial-0.1.0-rc.1.json @@ -0,0 +1,241 @@ +{ + "format_version": "0.1-draft", + "id": "trace-adversarial", + "version": "0.1.0-rc.1", + "status": "candidate", + "suite": { + "id": "promptsyntax-conformance", + "version": "0.1.0-rc.1" + }, + "profile": "trace-user-0-3-draft", + "evidence_unit": "paired-semantic-mutation", + "selection_rule": "all-negative-cases-with-trace-diagnostics", + "scoring": { + "control_acceptance": "baseline-must-pass-as-conformant", + "mutation_detection": "mutant-must-fail-with-exact-diagnostic-set", + "aggregation": "unweighted-case-count" + }, + "scope": { + "claim": "Measures whether the candidate Trace validator accepts each valid control and detects a paired semantic mutation with the exact specification-owned diagnostic set.", + "attacker_capabilities": [ + "Corrupt or omit fields in a stored Trace document", + "Duplicate identifiers or evidence records to create ambiguity", + "Present Trace claims that diverge from independently observed producer facts" + ], + "excluded": [ + "Cryptographic authenticity, receipt veracity, and key compromise", + "Malformed transport encodings and filesystem attacks", + "Resource exhaustion and implementation-specific performance limits", + "Semantics not frozen by the candidate profile" + ] + }, + "threat_classes": [ + { + "id": "assembly-provenance", + "description": "Mutations that make ordered prompt assembly or provenance pointers incomplete, ambiguous, or cyclic." + }, + { + "id": "content-integrity", + "description": "Mutations that make materialized content disagree with its declared availability, digest, or UTF-8 byte length." + }, + { + "id": "routing-resolution", + "description": "Mutations that conceal or contradict the requested, applied, fallback, or resolved entity." + }, + { + "id": "event-boundary", + "description": "Mutations that make turn events or tool-boundary outcomes ambiguous or inconsistent." + }, + { + "id": "independent-replay", + "description": "Mutations exposed only by comparing the Trace with independently observed request, routing, or artifact facts." + } + ], + "mutations": [ + { + "id": "A001", + "threat_class": "assembly-provenance", + "operator": "field-replacement", + "baseline_case": "r1-inline-segments-contiguous", + "mutant_case": "r1-segment-index-gap", + "controlled_paths": ["/trace/events/0/segments/0/i"], + "attack": "Change a segment index so the displayed order no longer proves byte-contiguous assembly.", + "intended_diagnostics": ["TRACE_R1_SEGMENT_INDEX_INVALID"] + }, + { + "id": "A002", + "threat_class": "assembly-provenance", + "operator": "field-replacement", + "baseline_case": "r2-injected-segment-step-valid", + "mutant_case": "r2-dangling-segment-step", + "controlled_paths": ["/trace/events/0/segments/0/step"], + "attack": "Point an injected segment at a provenance step that does not exist in its inference.", + "intended_diagnostics": ["TRACE_R2_STEP_NOT_FOUND"] + }, + { + "id": "A003", + "threat_class": "assembly-provenance", + "operator": "field-replacement", + "baseline_case": "r2-r3-injected-provenance-valid", + "mutant_case": "r3-provenance-cycle", + "controlled_paths": ["/trace/events/0/steps/0/parent"], + "attack": "Replace an acyclic provenance chain with mutually recursive parent pointers.", + "intended_diagnostics": ["TRACE_R3_STEP_CYCLE"] + }, + { + "id": "A004", + "threat_class": "assembly-provenance", + "operator": "duplication", + "baseline_case": "r1-inline-segments-contiguous", + "mutant_case": "tamper-duplicate-segment-id", + "controlled_paths": ["/trace/events/0/segments/1"], + "attack": "Reuse a stable segment identifier for a second segment in one inference.", + "intended_diagnostics": ["TRACE_SEGMENT_ID_DUPLICATE"] + }, + { + "id": "A005", + "threat_class": "content-integrity", + "operator": "field-omission", + "baseline_case": "r5-oversight-materialized-valid", + "mutant_case": "r5-oversight-materialization-missing", + "controlled_paths": ["/trace/events/0/segments/0/content/materialized"], + "attack": "Remove required materialized bytes while retaining an oversight materialized-export claim.", + "intended_diagnostics": ["TRACE_R5_MATERIALIZATION_REQUIRED"] + }, + { + "id": "A006", + "threat_class": "content-integrity", + "operator": "content-corruption", + "baseline_case": "r5-oversight-materialized-valid", + "mutant_case": "tamper-external-digest-mismatch", + "controlled_paths": ["/trace/events/0/segments/0/content/materialized/text"], + "attack": "Change materialized content without updating its declared SHA-256 digest.", + "intended_diagnostics": ["TRACE_CONTENT_DIGEST_MISMATCH"] + }, + { + "id": "A007", + "threat_class": "content-integrity", + "operator": "metadata-corruption", + "baseline_case": "r5-oversight-materialized-valid", + "mutant_case": "tamper-external-length-mismatch", + "controlled_paths": ["/trace/events/0/segments/0/content/length"], + "attack": "Change declared content length so it disagrees with the materialized UTF-8 bytes.", + "intended_diagnostics": ["TRACE_CONTENT_LENGTH_MISMATCH"] + }, + { + "id": "A008", + "threat_class": "routing-resolution", + "operator": "field-replacement", + "baseline_case": "entity-kept-strict-valid", + "mutant_case": "entity-kept-applied-mismatch", + "controlled_paths": ["/trace/events/0/routing/fill/0/applied"], + "attack": "Label a different applied model as kept under an exact strict request.", + "intended_diagnostics": ["TRACE_ENTITY_KEPT_MISMATCH", "TRACE_ROUTING_FILL_MISMATCH"] + }, + { + "id": "A009", + "threat_class": "routing-resolution", + "operator": "policy-conflict", + "baseline_case": "entity-substituted-best-effort-valid", + "mutant_case": "entity-substituted-under-strict", + "controlled_paths": [ + "/trace/events/0/routing/attempts/0/ref", + "/trace/events/0/routing/fill/0/policy", + "/trace/events/0/routing/policy", + "/trace/events/0/routing/requested" + ], + "attack": "Report an out-of-route model substitution under a strict policy.", + "intended_diagnostics": ["TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED"] + }, + { + "id": "A010", + "threat_class": "routing-resolution", + "operator": "field-replacement", + "baseline_case": "entity-authored-fallback-valid", + "mutant_case": "entity-fallback-step-does-not-match", + "controlled_paths": ["/trace/events/0/routing/fill/0/route_step"], + "attack": "Point fallback provenance at an attempt that was not the first successful legal route step.", + "intended_diagnostics": ["TRACE_ENTITY_FALLBACK_STEP_INVALID"] + }, + { + "id": "A011", + "threat_class": "routing-resolution", + "operator": "field-replacement", + "baseline_case": "entity-kept-strict-valid", + "mutant_case": "routing-bound-disagrees-with-fill", + "controlled_paths": ["/trace/events/0/routing/bound"], + "attack": "Change the top-level bound model while leaving the successful attempt and fill unchanged.", + "intended_diagnostics": ["TRACE_ROUTING_BOUND_MISMATCH", "TRACE_ROUTING_FILL_MISMATCH"] + }, + { + "id": "A012", + "threat_class": "routing-resolution", + "operator": "field-omission", + "baseline_case": "entity-kept-strict-valid", + "mutant_case": "resolution-missing-applied-binding", + "controlled_paths": ["/trace/events/0/resolution/bindings"], + "attack": "Remove the resolution binding for the entity reported as invoked.", + "intended_diagnostics": ["TRACE_RESOLUTION_BOUND_MISSING"] + }, + { + "id": "A013", + "threat_class": "event-boundary", + "operator": "duplication", + "baseline_case": "r1-inline-segments-contiguous", + "mutant_case": "turn-event-identifiers-unique", + "controlled_paths": ["/trace/events/1"], + "attack": "Reuse an inference identifier for a later boundary event in the same turn.", + "intended_diagnostics": ["TRACE_EVENT_ID_DUPLICATE"] + }, + { + "id": "A014", + "threat_class": "event-boundary", + "operator": "cross-source-divergence", + "baseline_case": "producer-request-and-bound-match", + "mutant_case": "producer-turn-mismatch", + "controlled_paths": ["/transcript/turn"], + "attack": "Pair a Trace with independently observed producer facts from another turn.", + "intended_diagnostics": ["TRACE_PRODUCER_TURN_MISMATCH"] + }, + { + "id": "A015", + "threat_class": "event-boundary", + "operator": "cross-source-divergence", + "baseline_case": "producer-boundary-outcome-match", + "mutant_case": "producer-boundary-outcome-mismatch", + "controlled_paths": ["/transcript/boundaries/0/outcome"], + "attack": "Report a failed tool boundary when the independent producer fact records success.", + "intended_diagnostics": ["TRACE_PRODUCER_BOUNDARY_MISMATCH"] + }, + { + "id": "A016", + "threat_class": "independent-replay", + "operator": "cross-source-divergence", + "baseline_case": "producer-request-and-bound-match", + "mutant_case": "producer-request-bytes-mismatch", + "controlled_paths": ["/transcript/inferences/0/actual_request_utf8"], + "attack": "Present assembled prompt bytes that differ from the request independently observed by the producer.", + "intended_diagnostics": ["TRACE_R1_REQUEST_BYTES_MISMATCH"] + }, + { + "id": "A017", + "threat_class": "independent-replay", + "operator": "cross-source-divergence", + "baseline_case": "producer-request-and-bound-match", + "mutant_case": "producer-bound-entity-mismatch", + "controlled_paths": ["/transcript/inferences/0/routing/bound"], + "attack": "Claim that the requested model ran when the independent producer fact identifies another model.", + "intended_diagnostics": ["TRACE_PRODUCER_BOUND_MISMATCH"] + }, + { + "id": "A018", + "threat_class": "independent-replay", + "operator": "content-corruption", + "baseline_case": "producer-external-artifact-valid", + "mutant_case": "producer-external-artifact-digest-tampered", + "controlled_paths": ["/transcript/external_artifacts/0/content_utf8"], + "attack": "Supply independent artifact bytes that no longer match the content digest referenced by the Trace.", + "intended_diagnostics": ["TRACE_CONTENT_DIGEST_MISMATCH"] + } + ] +} diff --git a/conformance/core-adapter-result.schema.json b/conformance/core-adapter-result.schema.json index 6f2f490..e6eff33 100644 --- a/conformance/core-adapter-result.schema.json +++ b/conformance/core-adapter-result.schema.json @@ -42,13 +42,7 @@ "output": { "type": "object", "additionalProperties": false, - "required": [ - "round_trip", - "data_plane", - "segments", - "directives", - "parser_diagnostics" - ], + "required": ["round_trip", "data_plane", "segments", "directives", "parser_diagnostics"], "properties": { "round_trip": { "type": "string" }, "data_plane": { "type": "string" }, diff --git a/conformance/diagnostic-registry.json b/conformance/diagnostic-registry.json index 389e16f..a73f9e1 100644 --- a/conformance/diagnostic-registry.json +++ b/conformance/diagnostic-registry.json @@ -1,6 +1,102 @@ { "format_version": "0.1-draft", "diagnostics": [ + { + "code": "ADVERSARIAL_BASELINE_INVALID", + "phase": "benchmark", + "severity": "error", + "description": "A mutation's paired control does not pass as conformant." + }, + { + "code": "ADVERSARIAL_BENCHMARK_INVALID", + "phase": "benchmark", + "severity": "error", + "description": "The adversarial benchmark document is missing or violates its closed contract." + }, + { + "code": "ADVERSARIAL_CASE_MISSING", + "phase": "benchmark", + "severity": "error", + "description": "A benchmark control or mutant case is absent from the active suite report." + }, + { + "code": "ADVERSARIAL_FORMAT_UNSUPPORTED", + "phase": "benchmark", + "severity": "error", + "description": "The adversarial benchmark uses an unsupported format version." + }, + { + "code": "ADVERSARIAL_METADATA_MISMATCH", + "phase": "benchmark", + "severity": "error", + "description": "Benchmark identity, version, status, or profile disagrees with the active suite." + }, + { + "code": "ADVERSARIAL_MUTANT_CASE_DUPLICATE", + "phase": "benchmark", + "severity": "error", + "description": "The benchmark scores the same mutant suite case more than once." + }, + { + "code": "ADVERSARIAL_MUTANT_COVERAGE_EXTRA", + "phase": "benchmark", + "severity": "error", + "description": "The benchmark scores a case outside its declared exhaustive selection rule." + }, + { + "code": "ADVERSARIAL_MUTANT_COVERAGE_MISSING", + "phase": "benchmark", + "severity": "error", + "description": "A suite case eligible under the benchmark selection rule is not scored." + }, + { + "code": "ADVERSARIAL_MUTANT_CONTRACT_INVALID", + "phase": "benchmark", + "severity": "error", + "description": "A mutant case does not declare the benchmark's exact expected diagnostic set." + }, + { + "code": "ADVERSARIAL_MUTATION_ID_DUPLICATE", + "phase": "benchmark", + "severity": "error", + "description": "An adversarial mutation identifier occurs more than once." + }, + { + "code": "ADVERSARIAL_MUTATION_SCOPE_MISMATCH", + "phase": "benchmark", + "severity": "error", + "description": "Observed differences between a control and mutant escape or fail to exercise the declared mutation paths." + }, + { + "code": "ADVERSARIAL_REPORT_INVALID", + "phase": "benchmark", + "severity": "error", + "description": "The generated adversarial report violates its report contract." + }, + { + "code": "ADVERSARIAL_SUITE_INVALID", + "phase": "benchmark", + "severity": "error", + "description": "The benchmark's underlying conformance suite did not pass." + }, + { + "code": "ADVERSARIAL_THREAT_CLASS_DUPLICATE", + "phase": "benchmark", + "severity": "error", + "description": "An adversarial threat-class identifier occurs more than once." + }, + { + "code": "ADVERSARIAL_THREAT_CLASS_EMPTY", + "phase": "benchmark", + "severity": "error", + "description": "A declared adversarial threat class contains no mutation." + }, + { + "code": "ADVERSARIAL_THREAT_CLASS_UNKNOWN", + "phase": "benchmark", + "severity": "error", + "description": "A mutation cites an undeclared adversarial threat class." + }, { "code": "CORPUS_BLOCKED_WITHOUT_REASON", "phase": "corpus", diff --git a/conformance/diagnostic-registry.schema.json b/conformance/diagnostic-registry.schema.json index c30b58b..1b7a242 100644 --- a/conformance/diagnostic-registry.schema.json +++ b/conformance/diagnostic-registry.schema.json @@ -17,7 +17,16 @@ "properties": { "code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]+$" }, "phase": { - "enum": ["io", "corpus", "schema", "instance", "semantic", "producer", "adapter"] + "enum": [ + "io", + "corpus", + "schema", + "instance", + "semantic", + "producer", + "adapter", + "benchmark" + ] }, "severity": { "enum": ["error", "warning"] }, "description": { "type": "string", "minLength": 1 } diff --git a/conformance/families/trace-boundary-tamper.json b/conformance/families/trace-boundary-tamper.json index b0dd302..f915a5d 100644 --- a/conformance/families/trace-boundary-tamper.json +++ b/conformance/families/trace-boundary-tamper.json @@ -87,6 +87,8 @@ "requirements": ["PS-TRACE-CONTENT-001", "PS-TRACE-R1"], "trace": "conformance/fixtures/trace-user-kept.json", "patch": [ + { "op": "replace", "path": "/tier", "value": "oversight" }, + { "op": "replace", "path": "/content_mode", "value": "materialized" }, { "op": "replace", "path": "/events/0/segments/0/content", @@ -113,6 +115,8 @@ "requirements": ["PS-TRACE-CONTENT-001", "PS-TRACE-R1"], "trace": "conformance/fixtures/trace-user-kept.json", "patch": [ + { "op": "replace", "path": "/tier", "value": "oversight" }, + { "op": "replace", "path": "/content_mode", "value": "materialized" }, { "op": "replace", "path": "/events/0/segments/0/content", diff --git a/conformance/families/trace-outcomes.json b/conformance/families/trace-outcomes.json index 064d56a..db817a8 100644 --- a/conformance/families/trace-outcomes.json +++ b/conformance/families/trace-outcomes.json @@ -227,8 +227,21 @@ }, { "op": "replace", - "path": "/events/0/resolution/bindings/0/bound", - "value": "model:example/atlas-mini@2026-08-01" + "path": "/events/0/resolution/bindings", + "value": [ + { + "ref": "@model:example/atlas-4@2026-08-01", + "bound": "model:example/atlas-4@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + }, + { + "ref": "@model:example/atlas-mini@2026-08-01", + "bound": "model:example/atlas-mini@2026-08-01", + "rule": "qualified-canonical-reference", + "ambiguity_surfaced": false + } + ] } ], "expected": { diff --git a/conformance/families/trace-structure.json b/conformance/families/trace-structure.json index cfef50b..5b4cc59 100644 --- a/conformance/families/trace-structure.json +++ b/conformance/families/trace-structure.json @@ -66,6 +66,31 @@ ], "expected": { "conformant": true, "diagnostics": [] } }, + { + "id": "r2-injected-segment-step-valid", + "description": "An injected segment resolves to a context-inclusion step in the same inference.", + "requirements": ["PS-TRACE-R2"], + "trace": "conformance/fixtures/trace-user-kept.json", + "patch": [ + { "op": "replace", "path": "/events/0/segments/0/origin", "value": "injected" }, + { + "op": "replace", + "path": "/events/0/segments/0/step", + "value": "step-injected-001" + }, + { + "op": "add", + "path": "/events/0/steps/-", + "value": { + "id": "step-injected-001", + "kind": "context-inclusion", + "parent": "step-user-001", + "tier": "user" + } + } + ], + "expected": { "conformant": true, "diagnostics": [] } + }, { "id": "r2-dangling-segment-step", "description": "An injected segment cannot point to a missing provenance step.", @@ -73,7 +98,17 @@ "trace": "conformance/fixtures/trace-user-kept.json", "patch": [ { "op": "replace", "path": "/events/0/segments/0/origin", "value": "injected" }, - { "op": "replace", "path": "/events/0/segments/0/step", "value": "step-missing" } + { "op": "replace", "path": "/events/0/segments/0/step", "value": "step-missing" }, + { + "op": "add", + "path": "/events/0/steps/-", + "value": { + "id": "step-injected-001", + "kind": "context-inclusion", + "parent": "step-user-001", + "tier": "user" + } + } ], "expected": { "conformant": false, @@ -86,6 +121,26 @@ "requirements": ["PS-TRACE-R3"], "trace": "conformance/fixtures/trace-user-kept.json", "patch": [ + { + "op": "replace", + "path": "/events/0/segments", + "value": [ + { + "i": 0, + "id": "segment-user-001", + "origin": "user-authored", + "content": { "state": "inline", "text": "Summarize " }, + "step": "step-user-001" + }, + { + "i": 1, + "id": "segment-context-001", + "origin": "injected", + "content": { "state": "inline", "text": "the report." }, + "step": "step-context-001" + } + ] + }, { "op": "replace", "path": "/events/0/steps", @@ -93,14 +148,14 @@ { "id": "step-user-001", "kind": "user-content", - "parent": "step-policy-001", + "parent": "step-context-001", "tier": "user" }, { - "id": "step-policy-001", - "kind": "policy-injection", + "id": "step-context-001", + "kind": "context-inclusion", "parent": "step-user-001", - "tier": "operator" + "tier": "user" } ] } diff --git a/conformance/implementations.lock.json b/conformance/implementations.lock.json index e5684a5..bd49c7a 100644 --- a/conformance/implementations.lock.json +++ b/conformance/implementations.lock.json @@ -9,17 +9,41 @@ "fixture": "tests/conformance.json", "runtime": { "name": "rust", "version": "1.97.0" }, "command": [ - "cargo", "run", "--quiet", "--release", "--locked", "--bin", - "ps-core-adapter", "--", "{cases}", "{commit}" + "cargo", + "run", + "--quiet", + "--release", + "--locked", + "--bin", + "ps-core-adapter", + "--", + "{cases}", + "{commit}" ], "stream_command": [ - "cargo", "run", "--quiet", "--release", "--locked", "--bin", - "ps-core-adapter", "--", "{cases}", "{commit}", "--jsonl" + "cargo", + "run", + "--quiet", + "--release", + "--locked", + "--bin", + "ps-core-adapter", + "--", + "{cases}", + "{commit}", + "--jsonl" ], "trace_adapter": "src/bin/ps-trace-producer.rs", "trace_command": [ - "cargo", "run", "--quiet", "--release", "--locked", "--bin", - "ps-trace-producer", "--", "{input}" + "cargo", + "run", + "--quiet", + "--release", + "--locked", + "--bin", + "ps-trace-producer", + "--", + "{input}" ] }, { @@ -30,9 +54,7 @@ "fixture": "test/conformance.json", "runtime": { "name": "bun", "version": "1.3.14" }, "command": ["bun", "run", "core-adapter", "{cases}", "{commit}"], - "stream_command": [ - "bun", "run", "core-adapter", "{cases}", "{commit}", "--jsonl" - ], + "stream_command": ["bun", "run", "core-adapter", "{cases}", "{commit}", "--jsonl"], "trace_adapter": "src/bin/trace-producer.ts", "trace_command": ["bun", "run", "trace-producer", "{input}"] } diff --git a/conformance/manifest.json b/conformance/manifest.json index 433c3b6..e0e3887 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -25,6 +25,10 @@ "profile": "conformance/profiles/trace-user-0.3-draft.json", "profile_schema": "conformance/profile.schema.json", "report_schema": "conformance/report.schema.json", + "adversarial_benchmark": "conformance/benchmarks/trace-adversarial-0.1.0-rc.1.json", + "adversarial_benchmark_schema": "conformance/adversarial-benchmark.schema.json", + "adversarial_report_schema": "conformance/adversarial-report.schema.json", + "adversarial_report": "conformance/reports/trace-adversarial-0.1.0-rc.1.json", "core_cases": "conformance/cases/core-parser.json", "core_cases_schema": "conformance/core-cases.schema.json", "core_differential_input_schema": "conformance/core-differential-input.schema.json", diff --git a/conformance/manifest.schema.json b/conformance/manifest.schema.json index 92b28b1..8a59d28 100644 --- a/conformance/manifest.schema.json +++ b/conformance/manifest.schema.json @@ -45,6 +45,10 @@ "profile", "profile_schema", "report_schema", + "adversarial_benchmark", + "adversarial_benchmark_schema", + "adversarial_report_schema", + "adversarial_report", "core_cases", "core_cases_schema", "core_differential_input_schema", @@ -73,6 +77,10 @@ "profile": { "type": "string", "minLength": 1 }, "profile_schema": { "type": "string", "minLength": 1 }, "report_schema": { "type": "string", "minLength": 1 }, + "adversarial_benchmark": { "type": "string", "minLength": 1 }, + "adversarial_benchmark_schema": { "type": "string", "minLength": 1 }, + "adversarial_report_schema": { "type": "string", "minLength": 1 }, + "adversarial_report": { "type": "string", "minLength": 1 }, "core_cases": { "type": "string", "minLength": 1 }, "core_cases_schema": { "type": "string", "minLength": 1 }, "core_differential_input_schema": { "type": "string", "minLength": 1 }, diff --git a/conformance/releases/0.1.0-rc.1.md b/conformance/releases/0.1.0-rc.1.md index 1f2421c..2bb59f5 100644 --- a/conformance/releases/0.1.0-rc.1.md +++ b/conformance/releases/0.1.0-rc.1.md @@ -5,7 +5,7 @@ been issued. ## Included evidence -- 34 executable cases in four Trace document and producer families. +- 35 executable cases in four Trace document and producer families. - Exact expected diagnostics for positive, negative, and tamper cases. - Complete coverage of the 22 requirements in `trace-user-0-3-draft`. - Independent producer transcripts for UTF-8 request bytes, invoked entities, boundary @@ -17,6 +17,10 @@ been issued. families, with zero expected answers embedded in generated inputs. - Ten real-producer adapter cases, including six independently replayed valid traces and four required typed rejections. +- An 18-mutation adversarial Trace benchmark with 10 unique passing controls, 5 threat + classes, exact-diagnostic scoring, machine-checked exhaustive selection over eligible + semantic-negative cases, and explicit exclusions. +- Linux and macOS snapshot lanes for cross-platform diagnostic and report stability. - A deterministic JSON report with no wall-clock fields. ## Reproduce @@ -24,6 +28,8 @@ been issued. ```bash cargo test --locked --manifest-path conformance/runner/Cargo.toml cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-suite . +cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- \ + run-adversarial-benchmark . cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- \ compare-core-adapters conformance/cases/core-parser.json \ /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json @@ -39,6 +45,6 @@ cargo run --quiet --release --locked --manifest-path conformance/runner/Cargo.to 1. Rebase onto the reviewed PR #4 result and run a requirement-level delta audit. 2. Accept or revise the four provisional semantic defaults in decision 13. 3. Require the Trace suite and both pinned Core adapter jobs to pass in CI. -4. Regenerate and review the deterministic report. +4. Regenerate and review the deterministic suite and adversarial reports. 5. Change suite, profile, and included family status only if the baseline is frozen. 6. Create the version tag and archive that immutable revision with the paper artifacts. diff --git a/conformance/reports/0.1.0-rc.1.json b/conformance/reports/0.1.0-rc.1.json index a9f8b17..e926c10 100644 --- a/conformance/reports/0.1.0-rc.1.json +++ b/conformance/reports/0.1.0-rc.1.json @@ -17,8 +17,8 @@ "conformant": true, "summary": { "families": 4, - "cases": 34, - "passed": 34, + "cases": 35, + "passed": 35, "failed": 0, "requirements_required": 22, "requirements_covered": 22 @@ -483,6 +483,17 @@ "PS-TRACE-R2" ] }, + { + "id": "r2-injected-segment-step-valid", + "passed": true, + "expected_conformant": true, + "actual_conformant": true, + "expected_diagnostics": [], + "actual_diagnostics": [], + "requirements": [ + "PS-TRACE-R2" + ] + }, { "id": "r2-r3-injected-provenance-valid", "passed": true, diff --git a/conformance/reports/trace-adversarial-0.1.0-rc.1.json b/conformance/reports/trace-adversarial-0.1.0-rc.1.json new file mode 100644 index 0000000..596aeca --- /dev/null +++ b/conformance/reports/trace-adversarial-0.1.0-rc.1.json @@ -0,0 +1,562 @@ +{ + "format_version": "0.1-draft", + "benchmark": { + "id": "trace-adversarial", + "version": "0.1.0-rc.1", + "status": "candidate" + }, + "suite": { + "id": "promptsyntax-conformance", + "version": "0.1.0-rc.1", + "status": "candidate" + }, + "profile": { + "id": "trace-user-0-3-draft", + "version": "0.1.0-rc.1", + "status": "candidate" + }, + "conformant": true, + "summary": { + "threat_classes": 5, + "unique_controls": 10, + "controls_accepted": 10, + "mutations": 18, + "controlled_mutations": 18, + "exactly_detected": 18, + "rejected_with_unexpected_diagnostics": 0, + "escaped": 0, + "invalid_controls": 0, + "missing_mutations": 0 + }, + "rates": { + "control_acceptance": { + "numerator": 10, + "denominator": 10 + }, + "exact_mutation_detection": { + "numerator": 18, + "denominator": 18 + } + }, + "coverage": { + "eligible_mutants": 18, + "scored_mutants": 18, + "missing": [], + "extra": [] + }, + "controls": [ + { + "case": "entity-authored-fallback-valid", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "entity-kept-strict-valid", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "entity-substituted-best-effort-valid", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "producer-boundary-outcome-match", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "producer-external-artifact-valid", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "producer-request-and-bound-match", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "r1-inline-segments-contiguous", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "r2-injected-segment-step-valid", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "r2-r3-injected-provenance-valid", + "accepted": true, + "actual_diagnostics": [] + }, + { + "case": "r5-oversight-materialized-valid", + "accepted": true, + "actual_diagnostics": [] + } + ], + "threat_classes": [ + { + "id": "assembly-provenance", + "description": "Mutations that make ordered prompt assembly or provenance pointers incomplete, ambiguous, or cyclic.", + "mutations": 4, + "exactly_detected": 4, + "rejected_with_unexpected_diagnostics": 0, + "escaped": 0, + "invalid_controls": 0, + "missing_mutations": 0, + "exact_detection_rate": { + "numerator": 4, + "denominator": 4 + } + }, + { + "id": "content-integrity", + "description": "Mutations that make materialized content disagree with its declared availability, digest, or UTF-8 byte length.", + "mutations": 3, + "exactly_detected": 3, + "rejected_with_unexpected_diagnostics": 0, + "escaped": 0, + "invalid_controls": 0, + "missing_mutations": 0, + "exact_detection_rate": { + "numerator": 3, + "denominator": 3 + } + }, + { + "id": "routing-resolution", + "description": "Mutations that conceal or contradict the requested, applied, fallback, or resolved entity.", + "mutations": 5, + "exactly_detected": 5, + "rejected_with_unexpected_diagnostics": 0, + "escaped": 0, + "invalid_controls": 0, + "missing_mutations": 0, + "exact_detection_rate": { + "numerator": 5, + "denominator": 5 + } + }, + { + "id": "event-boundary", + "description": "Mutations that make turn events or tool-boundary outcomes ambiguous or inconsistent.", + "mutations": 3, + "exactly_detected": 3, + "rejected_with_unexpected_diagnostics": 0, + "escaped": 0, + "invalid_controls": 0, + "missing_mutations": 0, + "exact_detection_rate": { + "numerator": 3, + "denominator": 3 + } + }, + { + "id": "independent-replay", + "description": "Mutations exposed only by comparing the Trace with independently observed request, routing, or artifact facts.", + "mutations": 3, + "exactly_detected": 3, + "rejected_with_unexpected_diagnostics": 0, + "escaped": 0, + "invalid_controls": 0, + "missing_mutations": 0, + "exact_detection_rate": { + "numerator": 3, + "denominator": 3 + } + } + ], + "mutations": [ + { + "id": "A001", + "threat_class": "assembly-provenance", + "operator": "field-replacement", + "baseline_case": "r1-inline-segments-contiguous", + "mutant_case": "r1-segment-index-gap", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/segments/0/i" + ], + "observed_paths": [ + "/trace/events/0/segments/0/i" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_R1_SEGMENT_INDEX_INVALID" + ], + "actual_diagnostics": [ + "TRACE_R1_SEGMENT_INDEX_INVALID" + ] + }, + { + "id": "A002", + "threat_class": "assembly-provenance", + "operator": "field-replacement", + "baseline_case": "r2-injected-segment-step-valid", + "mutant_case": "r2-dangling-segment-step", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/segments/0/step" + ], + "observed_paths": [ + "/trace/events/0/segments/0/step" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_R2_STEP_NOT_FOUND" + ], + "actual_diagnostics": [ + "TRACE_R2_STEP_NOT_FOUND" + ] + }, + { + "id": "A003", + "threat_class": "assembly-provenance", + "operator": "field-replacement", + "baseline_case": "r2-r3-injected-provenance-valid", + "mutant_case": "r3-provenance-cycle", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/steps/0/parent" + ], + "observed_paths": [ + "/trace/events/0/steps/0/parent" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_R3_STEP_CYCLE" + ], + "actual_diagnostics": [ + "TRACE_R3_STEP_CYCLE" + ] + }, + { + "id": "A004", + "threat_class": "assembly-provenance", + "operator": "duplication", + "baseline_case": "r1-inline-segments-contiguous", + "mutant_case": "tamper-duplicate-segment-id", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/segments/1" + ], + "observed_paths": [ + "/trace/events/0/segments/1" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_SEGMENT_ID_DUPLICATE" + ], + "actual_diagnostics": [ + "TRACE_SEGMENT_ID_DUPLICATE" + ] + }, + { + "id": "A005", + "threat_class": "content-integrity", + "operator": "field-omission", + "baseline_case": "r5-oversight-materialized-valid", + "mutant_case": "r5-oversight-materialization-missing", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/segments/0/content/materialized" + ], + "observed_paths": [ + "/trace/events/0/segments/0/content/materialized" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_R5_MATERIALIZATION_REQUIRED" + ], + "actual_diagnostics": [ + "TRACE_R5_MATERIALIZATION_REQUIRED" + ] + }, + { + "id": "A006", + "threat_class": "content-integrity", + "operator": "content-corruption", + "baseline_case": "r5-oversight-materialized-valid", + "mutant_case": "tamper-external-digest-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/segments/0/content/materialized/text" + ], + "observed_paths": [ + "/trace/events/0/segments/0/content/materialized/text" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ] + }, + { + "id": "A007", + "threat_class": "content-integrity", + "operator": "metadata-corruption", + "baseline_case": "r5-oversight-materialized-valid", + "mutant_case": "tamper-external-length-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/segments/0/content/length" + ], + "observed_paths": [ + "/trace/events/0/segments/0/content/length" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_CONTENT_LENGTH_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_CONTENT_LENGTH_MISMATCH" + ] + }, + { + "id": "A008", + "threat_class": "routing-resolution", + "operator": "field-replacement", + "baseline_case": "entity-kept-strict-valid", + "mutant_case": "entity-kept-applied-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/routing/fill/0/applied" + ], + "observed_paths": [ + "/trace/events/0/routing/fill/0/applied" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_ENTITY_KEPT_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_ENTITY_KEPT_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ] + }, + { + "id": "A009", + "threat_class": "routing-resolution", + "operator": "policy-conflict", + "baseline_case": "entity-substituted-best-effort-valid", + "mutant_case": "entity-substituted-under-strict", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/routing/attempts/0/ref", + "/trace/events/0/routing/fill/0/policy", + "/trace/events/0/routing/policy", + "/trace/events/0/routing/requested" + ], + "observed_paths": [ + "/trace/events/0/routing/attempts/0/ref", + "/trace/events/0/routing/fill/0/policy", + "/trace/events/0/routing/policy", + "/trace/events/0/routing/requested" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED" + ], + "actual_diagnostics": [ + "TRACE_ENTITY_SUBSTITUTION_NOT_AUTHORIZED" + ] + }, + { + "id": "A010", + "threat_class": "routing-resolution", + "operator": "field-replacement", + "baseline_case": "entity-authored-fallback-valid", + "mutant_case": "entity-fallback-step-does-not-match", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/routing/fill/0/route_step" + ], + "observed_paths": [ + "/trace/events/0/routing/fill/0/route_step" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_ENTITY_FALLBACK_STEP_INVALID" + ], + "actual_diagnostics": [ + "TRACE_ENTITY_FALLBACK_STEP_INVALID" + ] + }, + { + "id": "A011", + "threat_class": "routing-resolution", + "operator": "field-replacement", + "baseline_case": "entity-kept-strict-valid", + "mutant_case": "routing-bound-disagrees-with-fill", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/routing/bound" + ], + "observed_paths": [ + "/trace/events/0/routing/bound" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_ROUTING_BOUND_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_ROUTING_BOUND_MISMATCH", + "TRACE_ROUTING_FILL_MISMATCH" + ] + }, + { + "id": "A012", + "threat_class": "routing-resolution", + "operator": "field-omission", + "baseline_case": "entity-kept-strict-valid", + "mutant_case": "resolution-missing-applied-binding", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/0/resolution/bindings" + ], + "observed_paths": [ + "/trace/events/0/resolution/bindings/0" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_RESOLUTION_BOUND_MISSING" + ], + "actual_diagnostics": [ + "TRACE_RESOLUTION_BOUND_MISSING" + ] + }, + { + "id": "A013", + "threat_class": "event-boundary", + "operator": "duplication", + "baseline_case": "r1-inline-segments-contiguous", + "mutant_case": "turn-event-identifiers-unique", + "controlled_change": true, + "controlled_paths": [ + "/trace/events/1" + ], + "observed_paths": [ + "/trace/events/1" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_EVENT_ID_DUPLICATE" + ], + "actual_diagnostics": [ + "TRACE_EVENT_ID_DUPLICATE" + ] + }, + { + "id": "A014", + "threat_class": "event-boundary", + "operator": "cross-source-divergence", + "baseline_case": "producer-request-and-bound-match", + "mutant_case": "producer-turn-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/transcript/turn" + ], + "observed_paths": [ + "/transcript/turn" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_PRODUCER_TURN_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_PRODUCER_TURN_MISMATCH" + ] + }, + { + "id": "A015", + "threat_class": "event-boundary", + "operator": "cross-source-divergence", + "baseline_case": "producer-boundary-outcome-match", + "mutant_case": "producer-boundary-outcome-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/transcript/boundaries/0/outcome" + ], + "observed_paths": [ + "/transcript/boundaries/0/outcome" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_PRODUCER_BOUNDARY_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_PRODUCER_BOUNDARY_MISMATCH" + ] + }, + { + "id": "A016", + "threat_class": "independent-replay", + "operator": "cross-source-divergence", + "baseline_case": "producer-request-and-bound-match", + "mutant_case": "producer-request-bytes-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/transcript/inferences/0/actual_request_utf8" + ], + "observed_paths": [ + "/transcript/inferences/0/actual_request_utf8" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_R1_REQUEST_BYTES_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_R1_REQUEST_BYTES_MISMATCH" + ] + }, + { + "id": "A017", + "threat_class": "independent-replay", + "operator": "cross-source-divergence", + "baseline_case": "producer-request-and-bound-match", + "mutant_case": "producer-bound-entity-mismatch", + "controlled_change": true, + "controlled_paths": [ + "/transcript/inferences/0/routing/bound" + ], + "observed_paths": [ + "/transcript/inferences/0/routing/bound" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_PRODUCER_BOUND_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_PRODUCER_BOUND_MISMATCH" + ] + }, + { + "id": "A018", + "threat_class": "independent-replay", + "operator": "content-corruption", + "baseline_case": "producer-external-artifact-valid", + "mutant_case": "producer-external-artifact-digest-tampered", + "controlled_change": true, + "controlled_paths": [ + "/transcript/external_artifacts/0/content_utf8" + ], + "observed_paths": [ + "/transcript/external_artifacts/0/content_utf8" + ], + "status": "exact-detection", + "intended_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ], + "actual_diagnostics": [ + "TRACE_CONTENT_DIGEST_MISMATCH" + ] + } + ], + "diagnostics": [] +} diff --git a/conformance/runner/README.md b/conformance/runner/README.md index 5595dcb..296f28a 100644 --- a/conformance/runner/README.md +++ b/conformance/runner/README.md @@ -19,6 +19,9 @@ cargo run --manifest-path conformance/runner/Cargo.toml -- \ cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite . +cargo run --manifest-path conformance/runner/Cargo.toml -- \ + run-adversarial-benchmark . + cargo run --manifest-path conformance/runner/Cargo.toml -- \ compare-core-adapters conformance/cases/core-parser.json \ /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json @@ -57,6 +60,12 @@ producer, then validates the emitted Trace against both the Trace schema and tra Expected producer rejections must use the registered JSON error envelope and exit status `1`. +`run-adversarial-benchmark` scores specification-owned mutation cases against their paired +valid controls. Detection requires an exact diagnostic-set match. The deterministic report +separates exact detections, accepted escapes, unexpected diagnostics, invalid controls, and +missing cases overall and by threat class. It also materializes each pair and verifies that +every observed difference is covered by the mutation's declared JSON-pointer paths. + The runner performs no network requests and needs no provider SDK, model credentials, Python, Node, or Bun. Producing the TypeScript adapter document separately requires Bun. This is bootstrap tooling for a working draft, not a certification utility. diff --git a/conformance/runner/src/adversarial.rs b/conformance/runner/src/adversarial.rs new file mode 100644 index 0000000..b3eb435 --- /dev/null +++ b/conformance/runner/src/adversarial.rs @@ -0,0 +1,865 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::suite::{ + CaseReport, MaterializedCase, checked_schema, materialize_cases, read_json, validate_value, +}; +use crate::{Diagnostic, SuiteReport, run_suite}; + +const FORMAT_VERSION: &str = "0.1-draft"; + +#[derive(Debug, Serialize)] +pub struct AdversarialReport { + pub format_version: String, + pub benchmark: VersionedIdentity, + pub suite: VersionedIdentity, + pub profile: VersionedIdentity, + pub conformant: bool, + pub summary: AdversarialSummary, + pub rates: AdversarialRates, + pub coverage: AdversarialCoverage, + pub controls: Vec, + pub threat_classes: Vec, + pub mutations: Vec, + pub diagnostics: Vec, +} + +#[derive(Debug, Serialize)] +pub struct VersionedIdentity { + pub id: String, + pub version: String, + pub status: String, +} + +#[derive(Debug, Default, Serialize)] +pub struct AdversarialSummary { + pub threat_classes: usize, + pub unique_controls: usize, + pub controls_accepted: usize, + pub mutations: usize, + pub controlled_mutations: usize, + pub exactly_detected: usize, + pub rejected_with_unexpected_diagnostics: usize, + pub escaped: usize, + pub invalid_controls: usize, + pub missing_mutations: usize, +} + +#[derive(Debug, Default, Serialize)] +pub struct AdversarialRates { + pub control_acceptance: Ratio, + pub exact_mutation_detection: Ratio, +} + +#[derive(Debug, Default, Serialize)] +pub struct Ratio { + pub numerator: usize, + pub denominator: usize, +} + +#[derive(Debug, Default, Serialize)] +pub struct AdversarialCoverage { + pub eligible_mutants: usize, + pub scored_mutants: usize, + pub missing: Vec, + pub extra: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ControlReport { + pub case: String, + pub accepted: bool, + pub actual_diagnostics: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ThreatClassReport { + pub id: String, + pub description: String, + pub mutations: usize, + pub exactly_detected: usize, + pub rejected_with_unexpected_diagnostics: usize, + pub escaped: usize, + pub invalid_controls: usize, + pub missing_mutations: usize, + pub exact_detection_rate: Ratio, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum MutationStatus { + CaseMissing, + Escaped, + ExactDetection, + InvalidControl, + UnexpectedDiagnostics, +} + +#[derive(Debug, Serialize)] +pub struct MutationReport { + pub id: String, + pub threat_class: String, + pub operator: String, + pub baseline_case: String, + pub mutant_case: String, + pub controlled_change: bool, + pub controlled_paths: Vec, + pub observed_paths: Vec, + pub status: MutationStatus, + pub intended_diagnostics: Vec, + pub actual_diagnostics: Vec, +} + +#[derive(Debug, Deserialize)] +struct Benchmark { + format_version: String, + id: String, + version: String, + status: String, + suite: BenchmarkSuite, + profile: String, + threat_classes: Vec, + mutations: Vec, +} + +#[derive(Debug, Deserialize)] +struct BenchmarkSuite { + id: String, + version: String, +} + +#[derive(Debug, Deserialize)] +struct ThreatClass { + id: String, + description: String, +} + +#[derive(Debug, Deserialize)] +struct Mutation { + id: String, + threat_class: String, + operator: String, + baseline_case: String, + mutant_case: String, + controlled_paths: Vec, + intended_diagnostics: Vec, +} + +#[derive(Default)] +struct OutcomeCounts { + exact: usize, + unexpected: usize, + escaped: usize, + invalid_control: usize, + missing: usize, +} + +#[must_use] +pub fn run_adversarial_benchmark(root: &Path) -> AdversarialReport { + match run_adversarial_benchmark_inner(root) { + Ok(report) => report, + Err(diagnostic) => failure_report(diagnostic), + } +} + +#[allow(clippy::too_many_lines)] +fn run_adversarial_benchmark_inner(root: &Path) -> Result { + let manifest = read_json(root, "conformance/manifest.json")?; + let benchmark_path = manifest_file(&manifest, "adversarial_benchmark")?; + let benchmark_schema_path = manifest_file(&manifest, "adversarial_benchmark_schema")?; + let report_schema_path = manifest_file(&manifest, "adversarial_report_schema")?; + + let benchmark_schema = checked_schema(root, &benchmark_schema_path)?; + let benchmark_value = read_json(root, &benchmark_path)?; + let benchmark_validation = validate_value(&benchmark_schema, &benchmark_value); + if !benchmark_validation.conformant { + return Err(diagnostic( + "ADVERSARIAL_BENCHMARK_INVALID", + &benchmark_path, + benchmark_validation + .diagnostics + .first() + .map_or("benchmark schema validation failed", |item| { + item.message.as_str() + }), + )); + } + let benchmark: Benchmark = serde_json::from_value(benchmark_value).map_err(|error| { + diagnostic( + "ADVERSARIAL_BENCHMARK_INVALID", + &benchmark_path, + error.to_string(), + ) + })?; + + let suite = run_suite(root); + let mut diagnostics = Vec::new(); + if !suite.conformant { + push( + &mut diagnostics, + "ADVERSARIAL_SUITE_INVALID", + "/suite", + "the underlying conformance suite did not pass", + ); + } + validate_metadata(&benchmark, &suite, &mut diagnostics); + + let mut threat_class_ids = HashSet::new(); + let mut threat_classes = Vec::new(); + for (index, threat_class) in benchmark.threat_classes.iter().enumerate() { + if !threat_class_ids.insert(threat_class.id.as_str()) { + push( + &mut diagnostics, + "ADVERSARIAL_THREAT_CLASS_DUPLICATE", + format!("/threat_classes/{index}/id"), + format!("threat class {} occurs more than once", threat_class.id), + ); + } + threat_classes.push((threat_class.id.clone(), threat_class.description.clone())); + } + + let cases = index_cases(&suite); + let materialized_cases = materialize_cases(root)?; + let eligible_mutants = eligible_mutant_ids(&suite); + let control_ids = benchmark + .mutations + .iter() + .map(|mutation| mutation.baseline_case.clone()) + .collect::>(); + let mut missing_cases = HashSet::new(); + let controls = control_ids + .iter() + .map(|case_id| control_report(case_id, &cases, &mut missing_cases, &mut diagnostics)) + .collect::>(); + let accepted_controls = controls.iter().filter(|control| control.accepted).count(); + let accepted_by_id = controls + .iter() + .map(|control| (control.case.as_str(), control.accepted)) + .collect::>(); + + let mut mutation_ids = HashSet::new(); + let mut mutant_case_ids = HashSet::new(); + let mut mutation_reports = Vec::new(); + for (index, mutation) in benchmark.mutations.iter().enumerate() { + if !mutation_ids.insert(mutation.id.as_str()) { + push( + &mut diagnostics, + "ADVERSARIAL_MUTATION_ID_DUPLICATE", + format!("/mutations/{index}/id"), + format!("mutation id {} occurs more than once", mutation.id), + ); + } + if !mutant_case_ids.insert(mutation.mutant_case.as_str()) { + push( + &mut diagnostics, + "ADVERSARIAL_MUTANT_CASE_DUPLICATE", + format!("/mutations/{index}/mutant_case"), + format!( + "mutant case {} is scored more than once", + mutation.mutant_case + ), + ); + } + if !threat_class_ids.contains(mutation.threat_class.as_str()) { + push( + &mut diagnostics, + "ADVERSARIAL_THREAT_CLASS_UNKNOWN", + format!("/mutations/{index}/threat_class"), + format!( + "mutation {} cites unknown threat class {}", + mutation.id, mutation.threat_class + ), + ); + } + + let mut intended_diagnostics = mutation.intended_diagnostics.clone(); + intended_diagnostics.sort(); + let baseline_accepted = accepted_by_id + .get(mutation.baseline_case.as_str()) + .copied() + .unwrap_or(false); + let mutant = cases.get(mutation.mutant_case.as_str()).copied(); + if mutant.is_none() && missing_cases.insert(mutation.mutant_case.clone()) { + push( + &mut diagnostics, + "ADVERSARIAL_CASE_MISSING", + format!("/mutations/{index}/mutant_case"), + format!( + "case {} is absent from the suite report", + mutation.mutant_case + ), + ); + } + let actual_diagnostics = + mutant.map_or_else(Vec::new, |case| case.actual_diagnostics.clone()); + let mut controlled_paths = mutation.controlled_paths.clone(); + controlled_paths.sort(); + let (controlled_change, observed_paths) = mutation_scope( + &mutation.baseline_case, + &mutation.mutant_case, + &controlled_paths, + &materialized_cases, + ); + if cases.contains_key(mutation.baseline_case.as_str()) + && mutant.is_some() + && !controlled_change + { + push( + &mut diagnostics, + "ADVERSARIAL_MUTATION_SCOPE_MISMATCH", + format!("/mutations/{index}/controlled_paths"), + format!( + "observed differences for mutation {} do not match its declared controlled paths", + mutation.id + ), + ); + } + let status = score_mutation( + index, + mutation, + baseline_accepted, + mutant, + &intended_diagnostics, + &mut diagnostics, + ); + mutation_reports.push(MutationReport { + id: mutation.id.clone(), + threat_class: mutation.threat_class.clone(), + operator: mutation.operator.clone(), + baseline_case: mutation.baseline_case.clone(), + mutant_case: mutation.mutant_case.clone(), + controlled_change, + controlled_paths, + observed_paths, + status, + intended_diagnostics, + actual_diagnostics, + }); + } + + let scored_mutants = mutation_reports + .iter() + .map(|mutation| mutation.mutant_case.clone()) + .collect::>(); + let missing_mutant_coverage = eligible_mutants + .difference(&scored_mutants) + .cloned() + .collect::>(); + let extra_mutant_coverage = scored_mutants + .difference(&eligible_mutants) + .cloned() + .collect::>(); + for mutant in &missing_mutant_coverage { + push( + &mut diagnostics, + "ADVERSARIAL_MUTANT_COVERAGE_MISSING", + "/mutations", + format!("eligible mutant case {mutant} is not scored"), + ); + } + for mutant in &extra_mutant_coverage { + push( + &mut diagnostics, + "ADVERSARIAL_MUTANT_COVERAGE_EXTRA", + "/mutations", + format!("mutant case {mutant} is outside the declared selection rule"), + ); + } + + for (index, (id, _)) in threat_classes.iter().enumerate() { + if !mutation_reports + .iter() + .any(|mutation| mutation.threat_class == *id) + { + push( + &mut diagnostics, + "ADVERSARIAL_THREAT_CLASS_EMPTY", + format!("/threat_classes/{index}/id"), + format!("threat class {id} has no mutation"), + ); + } + } + + let counts = outcome_counts(&mutation_reports); + let threat_class_reports = threat_classes + .into_iter() + .map(|(id, description)| { + let class_mutations = mutation_reports + .iter() + .filter(|mutation| mutation.threat_class == id) + .collect::>(); + let class_counts = outcome_counts_refs(&class_mutations); + ThreatClassReport { + id, + description, + mutations: class_mutations.len(), + exactly_detected: class_counts.exact, + rejected_with_unexpected_diagnostics: class_counts.unexpected, + escaped: class_counts.escaped, + invalid_controls: class_counts.invalid_control, + missing_mutations: class_counts.missing, + exact_detection_rate: Ratio { + numerator: class_counts.exact, + denominator: class_mutations.len(), + }, + } + }) + .collect::>(); + + diagnostics.sort_by(|left, right| { + left.pointer + .cmp(&right.pointer) + .then_with(|| left.code.cmp(&right.code)) + .then_with(|| left.message.cmp(&right.message)) + }); + let all_exact = counts.exact == mutation_reports.len(); + let controlled_mutations = mutation_reports + .iter() + .filter(|mutation| mutation.controlled_change) + .count(); + let all_controlled = controlled_mutations == mutation_reports.len(); + let all_controls_accepted = accepted_controls == controls.len(); + let mut report = AdversarialReport { + format_version: FORMAT_VERSION.to_owned(), + benchmark: VersionedIdentity { + id: benchmark.id, + version: benchmark.version, + status: benchmark.status, + }, + suite: VersionedIdentity { + id: suite.suite.id, + version: suite.suite.version, + status: suite.suite.status, + }, + profile: VersionedIdentity { + id: suite.profile.id, + version: suite.profile.version, + status: suite.profile.status, + }, + conformant: all_exact + && all_controlled + && all_controls_accepted + && diagnostics.is_empty() + && suite.conformant, + summary: AdversarialSummary { + threat_classes: threat_class_reports.len(), + unique_controls: controls.len(), + controls_accepted: accepted_controls, + mutations: mutation_reports.len(), + controlled_mutations, + exactly_detected: counts.exact, + rejected_with_unexpected_diagnostics: counts.unexpected, + escaped: counts.escaped, + invalid_controls: counts.invalid_control, + missing_mutations: counts.missing, + }, + rates: AdversarialRates { + control_acceptance: Ratio { + numerator: accepted_controls, + denominator: controls.len(), + }, + exact_mutation_detection: Ratio { + numerator: counts.exact, + denominator: mutation_reports.len(), + }, + }, + coverage: AdversarialCoverage { + eligible_mutants: eligible_mutants.len(), + scored_mutants: scored_mutants.len(), + missing: missing_mutant_coverage, + extra: extra_mutant_coverage, + }, + controls, + threat_classes: threat_class_reports, + mutations: mutation_reports, + diagnostics, + }; + + let report_schema = checked_schema(root, &report_schema_path)?; + let report_value = serde_json::to_value(&report).map_err(|error| { + diagnostic( + "ADVERSARIAL_REPORT_INVALID", + &report_schema_path, + error.to_string(), + ) + })?; + let validation = validate_value(&report_schema, &report_value); + if !validation.conformant { + report.conformant = false; + report.diagnostics.push(diagnostic( + "ADVERSARIAL_REPORT_INVALID", + &report_schema_path, + validation + .diagnostics + .first() + .map_or("generated report does not satisfy its schema", |item| { + item.message.as_str() + }), + )); + } + Ok(report) +} + +fn manifest_file(manifest: &serde_json::Value, name: &str) -> Result { + manifest + .pointer(&format!("/files/{name}")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + diagnostic( + "ADVERSARIAL_BENCHMARK_INVALID", + format!("/files/{name}"), + "suite manifest does not declare the required adversarial artifact", + ) + }) +} + +fn validate_metadata( + benchmark: &Benchmark, + suite: &SuiteReport, + diagnostics: &mut Vec, +) { + if benchmark.format_version != FORMAT_VERSION { + push( + diagnostics, + "ADVERSARIAL_FORMAT_UNSUPPORTED", + "/format_version", + format!("benchmark format must be {FORMAT_VERSION}"), + ); + } + if benchmark.suite.id != suite.suite.id + || benchmark.suite.version != suite.suite.version + || benchmark.version != suite.suite.version + || benchmark.status != suite.suite.status + || benchmark.profile != suite.profile.id + { + push( + diagnostics, + "ADVERSARIAL_METADATA_MISMATCH", + "/", + "benchmark identity, suite version, status, or profile disagrees with the active suite", + ); + } +} + +fn index_cases(suite: &SuiteReport) -> HashMap<&str, &CaseReport> { + suite + .families + .iter() + .flat_map(|family| family.cases.iter()) + .map(|case| (case.id.as_str(), case)) + .collect() +} + +fn eligible_mutant_ids(suite: &SuiteReport) -> BTreeSet { + suite + .families + .iter() + .flat_map(|family| family.cases.iter()) + .filter(|case| { + !case.expected_conformant + && !case.expected_diagnostics.is_empty() + && case + .expected_diagnostics + .iter() + .all(|code| code.starts_with("TRACE_")) + }) + .map(|case| case.id.clone()) + .collect() +} + +fn mutation_scope( + baseline_case: &str, + mutant_case: &str, + controlled_paths: &[String], + cases: &HashMap, +) -> (bool, Vec) { + let Some(baseline) = cases.get(baseline_case) else { + return (false, Vec::new()); + }; + let Some(mutant) = cases.get(mutant_case) else { + return (false, Vec::new()); + }; + let baseline = composite_case(baseline); + let mutant = composite_case(mutant); + let mut observed = Vec::new(); + diff_paths(&baseline, &mutant, "", &mut observed); + observed.sort(); + observed.dedup(); + let all_observed_are_controlled = !observed.is_empty() + && observed.iter().all(|path| { + controlled_paths + .iter() + .any(|controlled| pointer_covers(controlled, path)) + }); + let every_controlled_path_changes = controlled_paths + .iter() + .all(|controlled| observed.iter().any(|path| pointer_covers(controlled, path))); + ( + all_observed_are_controlled && every_controlled_path_changes, + observed, + ) +} + +fn composite_case(case: &MaterializedCase) -> Value { + let mut object = serde_json::Map::new(); + object.insert("trace".to_owned(), case.trace.clone()); + object.insert( + "transcript".to_owned(), + case.transcript.clone().unwrap_or(Value::Null), + ); + Value::Object(object) +} + +fn diff_paths(left: &Value, right: &Value, pointer: &str, output: &mut Vec) { + if left == right { + return; + } + match (left, right) { + (Value::Object(left), Value::Object(right)) => { + let keys = left.keys().chain(right.keys()).collect::>(); + for key in keys { + let child = format!("{pointer}/{}", escape_pointer_token(key)); + match (left.get(key), right.get(key)) { + (Some(left), Some(right)) => diff_paths(left, right, &child, output), + _ => output.push(child), + } + } + } + (Value::Array(left), Value::Array(right)) => { + for index in 0..left.len().max(right.len()) { + let child = format!("{pointer}/{index}"); + match (left.get(index), right.get(index)) { + (Some(left), Some(right)) => diff_paths(left, right, &child, output), + _ => output.push(child), + } + } + } + _ => output.push(pointer.to_owned()), + } +} + +fn escape_pointer_token(value: &str) -> String { + value.replace('~', "~0").replace('/', "~1") +} + +fn pointer_covers(controlled: &str, observed: &str) -> bool { + observed == controlled + || observed + .strip_prefix(controlled) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +fn control_report( + case_id: &str, + cases: &HashMap<&str, &CaseReport>, + missing_cases: &mut HashSet, + diagnostics: &mut Vec, +) -> ControlReport { + let case = cases.get(case_id).copied(); + if case.is_none() && missing_cases.insert(case_id.to_owned()) { + push( + diagnostics, + "ADVERSARIAL_CASE_MISSING", + "/controls", + format!("baseline case {case_id} is absent from the suite report"), + ); + } + ControlReport { + case: case_id.to_owned(), + accepted: case.is_some_and(|report| { + report.passed + && report.expected_conformant + && report.actual_conformant + && report.actual_diagnostics.is_empty() + }), + actual_diagnostics: case.map_or_else(Vec::new, |report| report.actual_diagnostics.clone()), + } +} + +fn score_mutation( + index: usize, + mutation: &Mutation, + baseline_accepted: bool, + mutant: Option<&CaseReport>, + intended_diagnostics: &[String], + diagnostics: &mut Vec, +) -> MutationStatus { + let Some(mutant) = mutant else { + return MutationStatus::CaseMissing; + }; + if !baseline_accepted { + push( + diagnostics, + "ADVERSARIAL_BASELINE_INVALID", + format!("/mutations/{index}/baseline_case"), + format!( + "baseline case {} did not pass as conformant", + mutation.baseline_case + ), + ); + return MutationStatus::InvalidControl; + } + if mutant.expected_conformant || mutant.expected_diagnostics != intended_diagnostics { + push( + diagnostics, + "ADVERSARIAL_MUTANT_CONTRACT_INVALID", + format!("/mutations/{index}/mutant_case"), + format!( + "mutant case {} does not declare the benchmark's exact expected diagnostic set", + mutation.mutant_case + ), + ); + return MutationStatus::UnexpectedDiagnostics; + } + if mutant.actual_conformant { + MutationStatus::Escaped + } else if mutant.passed && mutant.actual_diagnostics == intended_diagnostics { + MutationStatus::ExactDetection + } else { + MutationStatus::UnexpectedDiagnostics + } +} + +fn outcome_counts(reports: &[MutationReport]) -> OutcomeCounts { + outcome_counts_refs(&reports.iter().collect::>()) +} + +fn outcome_counts_refs(reports: &[&MutationReport]) -> OutcomeCounts { + let mut counts = OutcomeCounts::default(); + for report in reports { + match report.status { + MutationStatus::ExactDetection => counts.exact += 1, + MutationStatus::UnexpectedDiagnostics => counts.unexpected += 1, + MutationStatus::Escaped => counts.escaped += 1, + MutationStatus::InvalidControl => counts.invalid_control += 1, + MutationStatus::CaseMissing => counts.missing += 1, + } + } + counts +} + +fn failure_report(diagnostic: Diagnostic) -> AdversarialReport { + let invalid = || VersionedIdentity { + id: "unknown".to_owned(), + version: "unknown".to_owned(), + status: "invalid".to_owned(), + }; + AdversarialReport { + format_version: FORMAT_VERSION.to_owned(), + benchmark: invalid(), + suite: invalid(), + profile: invalid(), + conformant: false, + summary: AdversarialSummary::default(), + rates: AdversarialRates::default(), + coverage: AdversarialCoverage::default(), + controls: Vec::new(), + threat_classes: Vec::new(), + mutations: Vec::new(), + diagnostics: vec![diagnostic], + } +} + +fn diagnostic(code: &str, pointer: impl Into, message: impl Into) -> Diagnostic { + Diagnostic { + code: code.to_owned(), + pointer: pointer.into(), + message: message.into(), + } +} + +fn push( + diagnostics: &mut Vec, + code: &str, + pointer: impl Into, + message: impl Into, +) { + diagnostics.push(diagnostic(code, pointer, message)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repository_benchmark_detects_every_mutation_exactly() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let report = run_adversarial_benchmark(&root); + assert!(report.conformant, "{:#?}", report.diagnostics); + assert_eq!(report.summary.mutations, 18); + assert_eq!(report.summary.controlled_mutations, 18); + assert_eq!(report.summary.exactly_detected, 18); + assert_eq!(report.summary.controls_accepted, 10); + assert_eq!(report.coverage.eligible_mutants, 18); + assert_eq!(report.coverage.scored_mutants, 18); + assert!(report.coverage.missing.is_empty()); + assert!(report.coverage.extra.is_empty()); + + let expected = std::fs::read_to_string( + root.join("conformance/reports/trace-adversarial-0.1.0-rc.1.json"), + ) + .expect("frozen adversarial report"); + let actual = format!( + "{}\n", + serde_json::to_string_pretty(&report).expect("serializable adversarial report") + ); + assert_eq!(actual, expected); + } + + #[test] + fn mutation_scoring_distinguishes_exact_detection_from_escape() { + let mutation = Mutation { + id: "A999".to_owned(), + threat_class: "test".to_owned(), + operator: "field-replacement".to_owned(), + baseline_case: "control".to_owned(), + mutant_case: "mutant".to_owned(), + controlled_paths: vec!["/trace/test".to_owned()], + intended_diagnostics: vec!["TRACE_TEST".to_owned()], + }; + let intended = vec!["TRACE_TEST".to_owned()]; + let exact = CaseReport { + id: "mutant".to_owned(), + passed: true, + expected_conformant: false, + actual_conformant: false, + expected_diagnostics: intended.clone(), + actual_diagnostics: intended.clone(), + requirements: Vec::new(), + }; + let mut diagnostics = Vec::new(); + assert_eq!( + score_mutation( + 0, + &mutation, + true, + Some(&exact), + &intended, + &mut diagnostics, + ), + MutationStatus::ExactDetection + ); + let escaped = CaseReport { + passed: false, + actual_conformant: true, + actual_diagnostics: Vec::new(), + ..exact + }; + assert_eq!( + score_mutation( + 0, + &mutation, + true, + Some(&escaped), + &intended, + &mut diagnostics, + ), + MutationStatus::Escaped + ); + assert!(diagnostics.is_empty()); + } +} diff --git a/conformance/runner/src/lib.rs b/conformance/runner/src/lib.rs index ad073a7..866654b 100644 --- a/conformance/runner/src/lib.rs +++ b/conformance/runner/src/lib.rs @@ -7,12 +7,14 @@ use jsonschema::PatternOptions; use serde::{Deserialize, Serialize}; use serde_json::Value; +mod adversarial; mod core; mod generator; mod producer; mod suite; mod trace; +pub use adversarial::{AdversarialReport, run_adversarial_benchmark}; pub use core::{CoreDifferentialReport, compare_core_adapter_streams, compare_core_adapters_json}; pub use generator::{GeneratorMetadata, write_generated_core_cases}; pub use producer::{TraceProducerAdapterReport, run_trace_producer_adapter}; diff --git a/conformance/runner/src/main.rs b/conformance/runner/src/main.rs index 7e531ea..e3ca398 100644 --- a/conformance/runner/src/main.rs +++ b/conformance/runner/src/main.rs @@ -7,9 +7,9 @@ use std::process::ExitCode; use promptsyntax_conformance::{ CheckReport, CoreDifferentialReport, ValidationReport, check_requirements_json, - check_schema_json, compare_core_adapter_streams, compare_core_adapters_json, run_suite, - run_trace_producer_adapter, validate_instance_json, validate_trace_producer_json, - write_generated_core_cases, + check_schema_json, compare_core_adapter_streams, compare_core_adapters_json, + run_adversarial_benchmark, run_suite, run_trace_producer_adapter, validate_instance_json, + validate_trace_producer_json, write_generated_core_cases, }; use serde::Serialize; @@ -194,6 +194,16 @@ fn main() -> ExitCode { let report = run_suite(std::path::Path::new(&root)); emit_report(&report, report.conformant) } + Some("run-adversarial-benchmark") => { + let Some(root) = args.next() else { + return usage(); + }; + if args.next().is_some() { + return usage(); + } + let report = run_adversarial_benchmark(std::path::Path::new(&root)); + emit_report(&report, report.conformant) + } Some("run-trace-producer-adapter") => { let Some(root) = args.next() else { return usage(); @@ -329,7 +339,7 @@ fn emit_report(report: &impl Serialize, conformant: bool) -> ExitCode { fn usage() -> ExitCode { eprintln!( - "usage:\n ps-conformance generate-core-differential \n ps-conformance check-requirements \n ps-conformance check-schema \n ps-conformance validate-instance \n ps-conformance validate-trace-producer \n ps-conformance run-suite \n ps-conformance run-trace-producer-adapter [args...]\n ps-conformance compare-core-adapters \n ps-conformance compare-core-streams " + "usage:\n ps-conformance generate-core-differential \n ps-conformance check-requirements \n ps-conformance check-schema \n ps-conformance validate-instance \n ps-conformance validate-trace-producer \n ps-conformance run-suite \n ps-conformance run-adversarial-benchmark \n ps-conformance run-trace-producer-adapter [args...]\n ps-conformance compare-core-adapters \n ps-conformance compare-core-streams " ); ExitCode::from(2) } diff --git a/conformance/runner/src/suite.rs b/conformance/runner/src/suite.rs index 812e75b..2f3244c 100644 --- a/conformance/runner/src/suite.rs +++ b/conformance/runner/src/suite.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fs; use std::path::{Component, Path, PathBuf}; @@ -112,6 +112,10 @@ struct ManifestFiles { profile: String, profile_schema: String, report_schema: String, + adversarial_benchmark: String, + adversarial_benchmark_schema: String, + adversarial_report_schema: String, + adversarial_report: String, core_cases: String, core_cases_schema: String, core_differential_input_schema: String, @@ -203,6 +207,12 @@ struct ProducerCorpusCase { expected: Value, } +#[derive(Debug)] +pub(crate) struct MaterializedCase { + pub trace: Value, + pub transcript: Option, +} + #[must_use] pub fn run_suite(root: &Path) -> SuiteReport { match run_suite_inner(root) { @@ -211,6 +221,54 @@ pub fn run_suite(root: &Path) -> SuiteReport { } } +pub(crate) fn materialize_cases( + root: &Path, +) -> Result, Diagnostic> { + let manifest_value = read_json(root, "conformance/manifest.json")?; + let manifest: Manifest = deserialize_value(manifest_value, "/conformance/manifest.json")?; + let mut materialized = HashMap::new(); + for family_entry in &manifest.families { + let family_value = read_json(root, &family_entry.path)?; + let family: CaseFamily = deserialize_value(family_value, &family_entry.path)?; + for case in family.cases { + let mut trace = read_json(root, &case.trace)?; + apply_patch(&mut trace, &case.patch).map_err(|message| { + diagnostic( + "CASE_PATCH_INVALID", + &family_entry.path, + format!("case {} trace patch failed: {message}", case.id), + ) + })?; + let transcript = case + .transcript + .as_deref() + .map(|path| { + let mut value = read_json(root, path)?; + apply_patch(&mut value, &case.transcript_patch).map_err(|message| { + diagnostic( + "CASE_PATCH_INVALID", + path, + format!("case {} transcript patch failed: {message}", case.id), + ) + })?; + Ok(value) + }) + .transpose()?; + if materialized + .insert(case.id.clone(), MaterializedCase { trace, transcript }) + .is_some() + { + return Err(diagnostic( + "SUITE_CASE_ID_DUPLICATE", + &family_entry.path, + format!("case id {} occurs more than once", case.id), + )); + } + } + } + Ok(materialized) +} + #[allow(clippy::too_many_lines)] fn run_suite_inner(root: &Path) -> Result { let manifest_value = read_json(root, "conformance/manifest.json")?; @@ -291,6 +349,10 @@ fn run_suite_inner(root: &Path) -> Result { checked_schema(root, &manifest.files.trace_producer_cases_schema)?; let profile_schema = checked_schema(root, &manifest.files.profile_schema)?; let report_schema = checked_schema(root, &manifest.files.report_schema)?; + let adversarial_benchmark_schema = + checked_schema(root, &manifest.files.adversarial_benchmark_schema)?; + let adversarial_report_schema = + checked_schema(root, &manifest.files.adversarial_report_schema)?; let core_cases_schema = checked_schema(root, &manifest.files.core_cases_schema)?; let _core_differential_input_schema = checked_schema(root, &manifest.files.core_differential_input_schema)?; @@ -328,6 +390,18 @@ fn run_suite_inner(root: &Path) -> Result { validate_value(&report_schema, &candidate_report), &manifest.files.candidate_report, ); + let adversarial_benchmark = read_json(root, &manifest.files.adversarial_benchmark)?; + append_validation( + &mut suite_diagnostics, + validate_value(&adversarial_benchmark_schema, &adversarial_benchmark), + &manifest.files.adversarial_benchmark, + ); + let adversarial_report = read_json(root, &manifest.files.adversarial_report)?; + append_validation( + &mut suite_diagnostics, + validate_value(&adversarial_report_schema, &adversarial_report), + &manifest.files.adversarial_report, + ); for report_path in [ &manifest.files.core_differential_report, &manifest.files.core_generated_differential_report, @@ -824,7 +898,7 @@ fn registry_codes(value: &Value, diagnostics: &mut Vec) -> HashSet Result { +pub(crate) fn checked_schema(root: &Path, relative: &str) -> Result { let schema = read_json(root, relative)?; ensure_schema(&schema, relative)?; Ok(schema) @@ -866,7 +940,7 @@ fn ensure_instance(schema: &Value, instance: &Value, pointer: &str) -> Result<() } } -fn validate_value(schema: &Value, instance: &Value) -> crate::ValidationReport { +pub(crate) fn validate_value(schema: &Value, instance: &Value) -> crate::ValidationReport { let schema_bytes = serde_json::to_vec(schema).unwrap_or_default(); let instance_bytes = serde_json::to_vec(instance).unwrap_or_default(); validate_instance_json(&schema_bytes, &instance_bytes) @@ -891,7 +965,7 @@ fn deserialize_value Deserialize<'de>>( .map_err(|error| diagnostic("SUITE_JSON_INVALID", pointer, error.to_string())) } -fn read_json(root: &Path, relative: &str) -> Result { +pub(crate) fn read_json(root: &Path, relative: &str) -> Result { let path = resolve_path(root, relative)?; let bytes = fs::read(&path) .map_err(|error| diagnostic("SUITE_FILE_READ_FAILED", relative, error.to_string()))?;