Conformance: add executable Trace suite candidate

.github/workflows/conformance.yml

@@ -0,0 +1,182 @@
+name: Conformance
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/conformance.yml"
+      - "conformance/**"
+      - "decisions/**"
+      - "examples/**"
+      - "schemas/**"
+      - "spec/**"
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/conformance.yml"
+      - "conformance/**"
+      - "decisions/**"
+      - "examples/**"
+      - "schemas/**"
+      - "spec/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  suite:
+    name: Trace suite
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout standard and corpus
+        uses: actions/checkout@v4
+
+      - name: Install stable Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.85.0
+          components: rustfmt, clippy
+
+      - name: Check Rust formatting
+        run: cargo fmt --manifest-path conformance/runner/Cargo.toml -- --check
+
+      - name: Run strict Rust lint
+        run: cargo clippy --manifest-path conformance/runner/Cargo.toml --all-targets -- -D warnings
+
+      - name: Run runner tests and suite
+        run: cargo test --locked --manifest-path conformance/runner/Cargo.toml
+
+      - name: Verify deterministic report snapshot
+        run: |
+          cargo run --quiet --locked --manifest-path conformance/runner/Cargo.toml -- run-suite . > /tmp/promptsyntax-conformance-report.json
+          cmp conformance/reports/0.1.0-rc.1.json /tmp/promptsyntax-conformance-report.json
+
+  cross-language-core:
+    name: Cross-language Core and Trace adapters
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout standard and canonical cases
+        uses: actions/checkout@v4
+        with:
+          path: standard
+
+      - name: Checkout pinned Rust implementation
+        uses: actions/checkout@v4
+        with:
+          repository: pathscale/PromptSyntax-rs
+          ref: 2789fac8acb810279ece887f46c025ef987ce0bf
+          path: implementations/promptsyntax-rs
+
+      - name: Checkout pinned TypeScript implementation
+        uses: actions/checkout@v4
+        with:
+          repository: pathscale/PromptSyntax-ts
+          ref: 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254
+          path: implementations/promptsyntax-ts
+
+      - name: Install stable Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.97.0
+          components: rustfmt, clippy
+
+      - name: Install Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Install TypeScript implementation dependencies
+        run: bun install --frozen-lockfile
+        working-directory: implementations/promptsyntax-ts
+
+      - name: Run Rust implementation checks against canonical cases
+        run: |
+          cp standard/conformance/cases/core-parser.json implementations/promptsyntax-rs/tests/conformance.json
+          cargo fmt --manifest-path implementations/promptsyntax-rs/Cargo.toml -- --check
+          cargo test --locked --all-targets --manifest-path implementations/promptsyntax-rs/Cargo.toml
+          cargo clippy --locked --all-targets --manifest-path implementations/promptsyntax-rs/Cargo.toml -- -D warnings
+
+      - name: Run TypeScript implementation checks against canonical cases
+        run: |
+          cp ../../standard/conformance/cases/core-parser.json test/conformance.json
+          bun test
+          bun run typecheck
+          bun run lint
+          bun run build
+        working-directory: implementations/promptsyntax-ts
+
+      - name: Produce canonical normalized Core results
+        run: |
+          cargo run --quiet --release --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml --bin ps-core-adapter -- \
+            standard/conformance/cases/core-parser.json 2789fac8acb810279ece887f46c025ef987ce0bf \
+            > /tmp/promptsyntax-rs-core.json
+          bun implementations/promptsyntax-ts/src/bin/core-adapter.ts \
+            standard/conformance/cases/core-parser.json 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 \
+            > /tmp/promptsyntax-ts-core.json
+
+      - name: Validate and compare canonical Core results
+        run: |
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/core-adapter-result.schema.json /tmp/promptsyntax-rs-core.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/core-adapter-result.schema.json /tmp/promptsyntax-ts-core.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            compare-core-adapters standard/conformance/cases/core-parser.json \
+            /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json \
+            > /tmp/core-differential.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/core-differential-report.schema.json /tmp/core-differential.json
+          cmp standard/conformance/reports/core-differential-0.1.0-rc.1.json /tmp/core-differential.json
+
+      - name: Generate deterministic 100,000-case differential input
+        run: |
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            generate-core-differential 100000 20270803 > /tmp/core-generated.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/core-differential-input.schema.json /tmp/core-generated.json
+
+      - name: Stream generated Core results from both implementations
+        run: |
+          cargo run --quiet --release --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml --bin ps-core-adapter -- \
+            /tmp/core-generated.json 2789fac8acb810279ece887f46c025ef987ce0bf --jsonl \
+            > /tmp/promptsyntax-rs-generated.jsonl
+          bun implementations/promptsyntax-ts/src/bin/core-adapter.ts \
+            /tmp/core-generated.json 3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 --jsonl \
+            > /tmp/promptsyntax-ts-generated.jsonl
+
+      - name: Compare all 100,000 generated normalized outputs
+        run: |
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            compare-core-streams 100000 20270803 \
+            /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl \
+            > /tmp/core-generated-differential.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/core-differential-report.schema.json \
+            /tmp/core-generated-differential.json
+          cmp standard/conformance/reports/core-generated-differential-0.1.0-rc.1.json \
+            /tmp/core-generated-differential.json
+
+      - name: Run both real Trace producers
+        run: |
+          cargo build --quiet --release --locked --manifest-path implementations/promptsyntax-rs/Cargo.toml \
+            --bin ps-trace-producer
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            run-trace-producer-adapter standard promptsyntax-rs 0.1.0 \
+            2789fac8acb810279ece887f46c025ef987ce0bf \
+            implementations/promptsyntax-rs/target/release/ps-trace-producer \
+            > /tmp/trace-producer-rs.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            run-trace-producer-adapter standard promptsyntax-ts 0.1.0 \
+            3c2c505d6f8fa836aa42abe12c0b466bcc3ba254 \
+            bun implementations/promptsyntax-ts/src/bin/trace-producer.ts \
+            > /tmp/trace-producer-ts.json
+
+      - name: Validate and freeze Trace producer reports
+        run: |
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/adapter-result.schema.json /tmp/trace-producer-rs.json
+          cargo run --quiet --release --locked --manifest-path standard/conformance/runner/Cargo.toml -- \
+            validate-instance standard/conformance/adapter-result.schema.json /tmp/trace-producer-ts.json
+          cmp standard/conformance/reports/trace-producer-rs-0.1.0-rc.1.json /tmp/trace-producer-rs.json
+          cmp standard/conformance/reports/trace-producer-ts-0.1.0-rc.1.json /tmp/trace-producer-ts.json

.gitignore

@@ -26,6 +26,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 .eslintcache
 .cache
 *.tsbuildinfo
+conformance/runner/target
 
 # LaTeX build artifacts (paper/)
 *.aux

README.md

@@ -52,12 +52,14 @@ anything.
 | [`examples/`](./examples/) | Machine-validated example trace |
 | [`decisions/`](./decisions/) | Design decision records (with rejected alternatives) |
 | [`profiles/`](./profiles/) | Externalized mappings: vendor, oversight, policy bindings |
+| [`conformance/`](./conformance/) | Language-independent requirements, vectors, schemas, reports, and Rust runner |
 
 ## Status
 
-**Draft v0.2.1, pre-implementation. Not yet stable.** A reference implementation and a
-formative multi-stakeholder study are in progress; an academic paper is in preparation.
-This is a proposal seeking scrutiny, not a finished standard.
+**Draft v0.2.1 with experimental implementations. Not yet stable.** Rust and TypeScript
+Core parsers and a candidate Trace conformance suite are under active review; a formative
+multi-stakeholder study and academic paper are in preparation. This is a proposal seeking
+scrutiny, not a finished standard.
 
 ## Design principles
 
@@ -72,6 +74,31 @@ Per-layer, versioned, and named, e.g. `PS/Core 0.2 + PS/Trace 0.2 (user tier,
 routing-complete/assembly-partial)`. **Unqualified conformance claims are
 non-conformant:** an implementation must state what it covers.
 
+The draft suite is self-contained in this repository. Its stable-Rust runner has no
+provider, model, Node, or Python dependency:
+
+```sh
+cargo test --manifest-path conformance/runner/Cargo.toml
+cargo run --manifest-path conformance/runner/Cargo.toml -- \
+  check-requirements conformance/requirements.json
+cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite .
+cargo run --manifest-path conformance/runner/Cargo.toml -- \
+  compare-core-adapters conformance/cases/core-parser.json \
+  /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json
+cargo run --release --manifest-path conformance/runner/Cargo.toml -- \
+  generate-core-differential 100000 20270803 > /tmp/core-generated.json
+cargo run --release --manifest-path conformance/runner/Cargo.toml -- \
+  compare-core-streams 100000 20270803 \
+  /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl
+```
+
+Candidate `0.1.0-rc.1` currently executes 34 Trace document and producer cases with full
+coverage of its explicitly named 22-requirement profile. It also carries 101 Core parser
+vectors, compares full normalized Rust and TypeScript outputs, runs a deterministic
+100,000-input grammar differential, and runs ten real Trace producer cases against
+independent transcripts. Candidate status is not a released or unqualified
+implementation-conformance claim.
+
 ## Contributing
 
 This is a draft seeking exactly the scrutiny that hardens a specification. Feedback is

biome.json

@@ -60,6 +60,15 @@
     }
   },
   "files": {
-    "includes": ["**", "!dist", "!node_modules", "!*.config.js", "!*.config.ts", "!.claude"]
+    "includes": [
+      "**",
+      "!dist",
+      "!node_modules",
+      "!*.config.js",
+      "!*.config.ts",
+      "!.claude",
+      "!conformance/reports",
+      "!conformance/runner/target"
+    ]
   }
 }

conformance/README.md

@@ -0,0 +1,94 @@
+# PromptSyntax conformance corpus
+
+This directory contains the canonical, language-independent PromptSyntax conformance
+corpus and its repository-owned stable-Rust runner. The specification and expected vectors
+are the oracle. Implementations consume the corpus but do not define its expected outcomes.
+
+## Current status
+
+Version `0.1.0-rc.1` is a candidate suite against the specification draft at commit
+`7456634817f4eb68c8909a58b476505e6fd7c063`. It executes 34 document and producer cases
+across four Trace families and covers all 22 requirements in its named candidate profile.
+It also owns 101 Core parser vectors and a full normalized-output differential protocol for
+the Rust and TypeScript implementations. A deterministic grammar generator adds a
+100,000-input differential lane without treating either parser as the expected-answer
+oracle. A separate ten-case executable adapter corpus tests real Trace producers in both
+languages against independent transcripts. It is not a released conformance suite and must
+not be used for an unqualified claim.
+
+The cases freeze provisional v0.3 decisions for review without pretending those decisions
+are already normative. The machine-readable inventory retains blocked, proposed, and open
+entries so unresolved specification gaps remain visible rather than being silently omitted.
+
+## Ownership
+
+- `promptsyntax.org` owns specifications, schemas, requirements, vectors, the stable-Rust
+  runner, and published reports so the complete suite is versioned atomically.
+- `PromptSyntax-rs` and `PromptSyntax-ts` are implementations under test, not sources of
+  expected answers.
+- Each implementation exposes a thin adapter that contains no expected answers.
+
+The normative runner does not require Python, Node, Bun, a provider SDK, a network
+connection, or model credentials. The TypeScript implementation may be invoked separately
+with Bun when producing cross-language evidence.
+
+Run the repository-owned checker from the repository root:
+
+```bash
+cargo run --manifest-path conformance/runner/Cargo.toml -- \
+  check-requirements conformance/requirements.json
+
+cargo run --manifest-path conformance/runner/Cargo.toml -- run-suite .
+
+cargo run --manifest-path conformance/runner/Cargo.toml -- \
+  compare-core-adapters conformance/cases/core-parser.json \
+  /tmp/promptsyntax-rs-core.json /tmp/promptsyntax-ts-core.json
+
+cargo run --release --manifest-path conformance/runner/Cargo.toml -- \
+  generate-core-differential 100000 20270803 > /tmp/core-generated.json
+
+cargo run --release --manifest-path conformance/runner/Cargo.toml -- \
+  compare-core-streams 100000 20270803 \
+  /tmp/promptsyntax-rs-generated.jsonl /tmp/promptsyntax-ts-generated.jsonl
+
+cargo run --manifest-path conformance/runner/Cargo.toml -- \
+  run-trace-producer-adapter . promptsyntax-rs 0.1.0 <commit> \
+  /path/to/ps-trace-producer
+```
+
+The command emits the deterministic report stored in
+`conformance/reports/0.1.0-rc.1.json`. CI regenerates it and requires a byte-for-byte match.
+
+## Layout
+
+- `profiles/` freezes the named scope and candidate semantic decisions.
+- `families/` contains language-independent cases and expected diagnostics.
+- `fixtures/` contains reusable Trace and deterministic transcript inputs.
+- `cases/core-parser.json` is the single 101-case Core fixture consumed by both adapters.
+- `core-differential-input.schema.json` freezes the generator-to-adapter input shape.
+- `core-adapter-result.schema.json` freezes the normalized cross-language output shape.
+- `core-differential-report.schema.json` freezes the independent comparison report shape.
+- `cases/trace-producer.json` lists six valid and four rejection cases for real producers.
+- `trace-producer-input.schema.json` defines deterministic facts without expected output.
+- `trace-producer-cases.schema.json` freezes the external producer adapter protocol.
+- `reports/` contains deterministic, machine-readable evidence snapshots.
+- `runner/` contains the independent stable-Rust checker.
+- `adapters/` defines the implementation boundary and forbids embedded expected answers.
+
+## Conformance targets
+
+1. **Corpus:** metadata, references, artifacts, hashes, and coverage are internally valid.
+2. **Core parser:** source produces exact lossless segments, projections, spans, directives,
+   and diagnostics.
+3. **Trace document:** a trace and declared artifacts satisfy structural and semantic rules.
+4. **Trace producer:** deterministic execution facts produce a trace consistent with those
+   facts.
+
+Passing document validation does not prove that a deployed venue reported execution facts
+truthfully. Independent collection or attestation is a separate trust mechanism.
+
+## Development rule
+
+If the specification does not determine an expected result, do not add a normative vector.
+Record the ambiguity, resolve it in the specification and decision log, and only then freeze
+the expected outcome.

conformance/adapter-result.schema.json

@@ -0,0 +1,39 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://promptsyntax.org/conformance/adapter-result/0.1-draft",
+  "title": "PromptSyntax implementation adapter result",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["format_version", "target", "implementation", "results"],
+  "properties": {
+    "format_version": { "const": "0.1-draft" },
+    "target": { "enum": ["core-parser", "trace-producer"] },
+    "implementation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "version", "commit"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 },
+        "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }
+      }
+    },
+    "results": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["case_id", "conformant", "diagnostics", "output"],
+        "properties": {
+          "case_id": { "type": "string", "minLength": 1 },
+          "conformant": { "type": "boolean" },
+          "diagnostics": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "output": true
+        }
+      }
+    }
+  }
+}