From 0cad940af7fa79bf81b537e46b7afe0007faa36e Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 31 Jul 2026 00:38:05 +0200 Subject: [PATCH 1/6] Add catalog plan versus direct brief scaffold --- CATALOG.md | 6 +- cells/catalog-plan-vs-direct-brief/README.md | 32 ++++ .../catalog-plan-vs-direct-brief.kdl | 39 +++++ .../catalog/plans/receipt-report/lineage.tsv | 3 + .../plans/receipt-report/receipts/README.md | 6 + .../plans/receipt-report/versions/0000.md | 37 ++++ .../plans/receipt-report/versions/0001.md | 41 +++++ .../fixture/arm-b/inbox/brief-0000.md | 37 ++++ .../fixture/arm-b/inbox/brief-0001.md | 41 +++++ .../fixture/arm-b/inbox/thread.tsv | 3 + .../fixture/arm-b/receipts/README.md | 6 + .../fixture/arms.tsv | 3 + .../fixture/blockers.tsv | 3 + .../fixture/check-fixtures.sh | 160 ++++++++++++++++++ .../fixture/experiment.tsv | 15 ++ .../fixture/judges/judge-task.sh | 34 ++++ .../fixture/judges/report.test.mjs | 111 ++++++++++++ .../fixture/measurement-schema.tsv | 11 ++ .../fixture/provenance.tsv | 24 +++ .../fixture/receipt-schema.tsv | 18 ++ .../fixture/reference/report.mjs | 76 +++++++++ .../fixture/run-recovery-matrix.sh | 101 +++++++++++ .../fixture/scenario.tsv | 7 + .../fixture/task-repo/README.md | 11 ++ .../fixture/task-repo/package.json | 9 + .../fixture/task-repo/src/report.mjs | 13 ++ .../fixture/task-repo/test/report.test.mjs | 54 ++++++ evidence/harness-exclusions.tsv | 1 + 28 files changed, 900 insertions(+), 2 deletions(-) create mode 100644 cells/catalog-plan-vs-direct-brief/README.md create mode 100644 cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0000.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0001.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0000.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0001.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arms.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv create mode 100755 cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv create mode 100755 cells/catalog-plan-vs-direct-brief/fixture/judges/judge-task.sh create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/judges/report.test.mjs create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/reference/report.mjs create mode 100755 cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/task-repo/README.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/task-repo/package.json create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/task-repo/src/report.mjs create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/task-repo/test/report.test.mjs diff --git a/CATALOG.md b/CATALOG.md index 58cd2eb1..702e7867 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,7 +1,7 @@ # Current eval catalog and evidence -This is the sole current corpus surface: **45 included cells** (14 model-free, +This is the sole current corpus surface: **46 included cells** (15 model-free, 31 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active KDL and executable judge scripts. Latest accepted PASS evidence remains distinct from the append-only run history: a failed last run is visible without being advertised as accepted evidence, and cells with no @@ -16,6 +16,7 @@ harness-native loader plus canonical hook file. | Cell | Harness | Model(s) / effort | Model seats | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | |---|---|---|---:|---|---|---:|---|---| +| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 9 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | @@ -81,6 +82,7 @@ while the last-run column makes a recorded failure distinct from a cell with no ## Harness hook exclusions +- `catalog-plan-vs-direct-brief` / `cell` — deterministic plan-versus-brief fixture and recovery contract; no harness seat - `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness seat - `docs` / `judge:cold-reader` — one-shot offline Claude print grader; no bus identity, DING, or hook surface - `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness seat @@ -97,7 +99,7 @@ while the last-run column makes a recorded failure distinct from a cell with no - `targeted-reconcile-isolation` / `cell` — deterministic selected reconcile isolation probe; no bus-connected agent - `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness seat -These **15** rows are generated from structured exclusions. A cell-level row is required for +These **16** rows are generated from structured exclusions. A cell-level row is required for every derived model-free cell. The one-shot docs cold reader is a model-backed grader, but not a bus-connected agent: it has no st2 identity, DING sidecar, or lifecycle hooks. diff --git a/cells/catalog-plan-vs-direct-brief/README.md b/cells/catalog-plan-vs-direct-brief/README.md new file mode 100644 index 00000000..ccf90135 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/README.md @@ -0,0 +1,32 @@ +# catalog-plan-vs-direct-brief + +Model-free contract for the first experiment proposed by +[`st2 plans: today → dream`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb): +compare one realistic repository task delivered as a versioned local catalog +plan (A) with the same task delivered as an ordinary durable direct brief (B). + +The frozen task, repository, tools, budgets, intent revisions, judges, and done +condition are identical. The two intent documents are byte-identical at each +revision. Both arms receive durable local input and use the same evaluator-owned +receipt fields, so direct planning remains a valid control rather than an +intentionally fragile baseline. + +The deterministic fixture proves: + +- the seeded repository needs a real implementation and the arm-neutral + reference solution passes public and held-out correctness tests; +- both arms recover the exact initial intent after a cold restart while their + remote source is unavailable; +- both arms receive and recover the exact steered intent; +- correctness, restart, partition, and steering are gating outcomes, while + coordination traffic and model cost remain reported comparison metrics; and +- no model or provider is launched. + +Arm A reserves a catalog-local plan directory with immutable Markdown versions, +lineage metadata, and receipt space. Arm B reserves equivalent durable +message-thread metadata and receipt space. The scaffold deliberately does not +invent `plan.kdl` syntax or st2 runtime behavior. Those assertions remain +blocked until the st2 owner supplies one exact experimental source head, +supported interface, reproducible artifact hash, and focused receipts. A +paid/live A/B is a separate authorization with an exact model, effort, budget, +run order, and rollback. diff --git a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl new file mode 100644 index 00000000..f1b16695 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl @@ -0,0 +1,39 @@ +// Neutral, model-free fixture contract for a catalog-plan versus direct-brief A/B. +eval { + copy "./fixture" + max-timeout "60s" + + run "fixture-contract" { + command "bash ./check-fixtures.sh" + } + + judges { + judge "TASK — frozen repository starts red and its arm-neutral reference implementation passes" { + exec "grep -Fqx TASK-CONTRACT-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "PARITY — task intent, tools, budgets, judges, and done condition match across arms" { + exec "grep -Fqx ARM-PARITY-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "RESTART — both durable inputs recover the exact accepted revision after cold state loss" { + exec "grep -Fqx COLD-RESTART-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "PARTITION — recovery uses each arm's local durable source with its remote source unavailable" { + exec "grep -Fqx LOCAL-PARTITION-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "STEERING — a complete new intent version and a direct follow-up carry identical revised intent" { + exec "grep -Fqx HUMAN-STEERING-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "MEASUREMENT — correctness gates are separate from neutral traffic and cost observations" { + exec "grep -Fqx MEASUREMENT-CONTRACT-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "PROVENANCE — every authoritative input has a verified content hash" { + exec "grep -Fqx PROVENANCE-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "BOUNDARY — unsupported plan runtime and paid execution remain explicit blockers" { + exec "grep -Fqx PRODUCT-BLOCKERS-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "HERMETIC — the scaffold launches no model, provider, network client, or product runtime" { + exec "grep -Fqx HERMETIC-SCAFFOLD-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + } +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv new file mode 100644 index 00000000..ed50f7c3 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv @@ -0,0 +1,3 @@ +version parent reason body_sha256 +0000 - initial frozen repository task 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +0001 0000 human steering tightens input validation without changing scope aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md new file mode 100644 index 00000000..6f521b6b --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md @@ -0,0 +1,6 @@ +# Evaluator receipt destination + +Future A-arm runs write evaluator-owned receipts here using +`receipt-schema.tsv`. These files report observed experiment outcomes; they do +not define plan intent, grant authority, or imply that this model-free scaffold +ran an agent. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0000.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0000.md new file mode 100644 index 00000000..7f75d24e --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0000.md @@ -0,0 +1,37 @@ +# Receipt report: latest accepted pass and last run + +Revision: `0000` + +## Goal + +Implement the report core in `src/report.mjs`. The report must retain the most +recent accepted `PASS` for each cell independently from that cell's last run, +which may be a failure. + +## Scope and owner + +You own the frozen `receipt-report` repository for this task. Change only +`src/report.mjs`. + +## Required behavior + +- `parseJsonLines(text)` ignores blank lines and returns records in input order. +- Malformed JSON, non-object JSON, or a record missing non-empty `run_id`, + `cell`, or `result` fields throws a `TypeError` that identifies the input line. +- `summarize(records)` returns one row per cell, sorted by cell name. +- Each row is `{ cell, accepted_pass, last_run }`. +- `accepted_pass` is the complete most recent `PASS` record for the cell, or + `null` when the cell has no pass. +- `last_run` is the complete last record for the cell. +- Neither exported function mutates caller-owned records. + +## Invariants and allowed actions + +Use only the repository's existing Node, Bash, and Git tools. Do not access the +network or add dependencies. Do not weaken or replace tests. No action outside +the frozen repository is authorized. + +## Evidence and done condition + +Run `npm test`. Completion additionally requires the evaluator's held-out tests +to pass and a diff containing only `src/report.mjs`. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0001.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0001.md new file mode 100644 index 00000000..299b7a50 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0001.md @@ -0,0 +1,41 @@ +# Receipt report: latest accepted pass and last run + +Revision: `0001` + +## Goal + +Implement the report core in `src/report.mjs`. The report must retain the most +recent accepted `PASS` for each cell independently from that cell's last run, +which may be a failure. + +## Scope and owner + +You own the frozen `receipt-report` repository for this task. Change only +`src/report.mjs`. + +## Required behavior + +- `parseJsonLines(text)` ignores blank lines and returns records in input order. +- Malformed JSON, non-object JSON, or a record missing non-empty `run_id`, + `cell`, or `result` fields throws a `TypeError` that identifies the input line. +- Only `PASS` and `FAIL` results are valid. An unknown result throws a + `TypeError` that identifies its `run_id`. +- A duplicate `run_id` is invalid. Both `parseJsonLines(text)` and + `summarize(records)` throw a `TypeError` that identifies the duplicate. +- `summarize(records)` returns one row per cell, sorted by cell name. +- Each row is `{ cell, accepted_pass, last_run }`. +- `accepted_pass` is the complete most recent `PASS` record for the cell, or + `null` when the cell has no pass. +- `last_run` is the complete last record for the cell. +- Neither exported function mutates caller-owned records. + +## Invariants and allowed actions + +Use only the repository's existing Node, Bash, and Git tools. Do not access the +network or add dependencies. Do not weaken or replace tests. No action outside +the frozen repository is authorized. + +## Evidence and done condition + +Run `npm test`. Completion additionally requires the evaluator's held-out tests +to pass and a diff containing only `src/report.mjs`. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0000.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0000.md new file mode 100644 index 00000000..7f75d24e --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0000.md @@ -0,0 +1,37 @@ +# Receipt report: latest accepted pass and last run + +Revision: `0000` + +## Goal + +Implement the report core in `src/report.mjs`. The report must retain the most +recent accepted `PASS` for each cell independently from that cell's last run, +which may be a failure. + +## Scope and owner + +You own the frozen `receipt-report` repository for this task. Change only +`src/report.mjs`. + +## Required behavior + +- `parseJsonLines(text)` ignores blank lines and returns records in input order. +- Malformed JSON, non-object JSON, or a record missing non-empty `run_id`, + `cell`, or `result` fields throws a `TypeError` that identifies the input line. +- `summarize(records)` returns one row per cell, sorted by cell name. +- Each row is `{ cell, accepted_pass, last_run }`. +- `accepted_pass` is the complete most recent `PASS` record for the cell, or + `null` when the cell has no pass. +- `last_run` is the complete last record for the cell. +- Neither exported function mutates caller-owned records. + +## Invariants and allowed actions + +Use only the repository's existing Node, Bash, and Git tools. Do not access the +network or add dependencies. Do not weaken or replace tests. No action outside +the frozen repository is authorized. + +## Evidence and done condition + +Run `npm test`. Completion additionally requires the evaluator's held-out tests +to pass and a diff containing only `src/report.mjs`. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0001.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0001.md new file mode 100644 index 00000000..299b7a50 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/brief-0001.md @@ -0,0 +1,41 @@ +# Receipt report: latest accepted pass and last run + +Revision: `0001` + +## Goal + +Implement the report core in `src/report.mjs`. The report must retain the most +recent accepted `PASS` for each cell independently from that cell's last run, +which may be a failure. + +## Scope and owner + +You own the frozen `receipt-report` repository for this task. Change only +`src/report.mjs`. + +## Required behavior + +- `parseJsonLines(text)` ignores blank lines and returns records in input order. +- Malformed JSON, non-object JSON, or a record missing non-empty `run_id`, + `cell`, or `result` fields throws a `TypeError` that identifies the input line. +- Only `PASS` and `FAIL` results are valid. An unknown result throws a + `TypeError` that identifies its `run_id`. +- A duplicate `run_id` is invalid. Both `parseJsonLines(text)` and + `summarize(records)` throw a `TypeError` that identifies the duplicate. +- `summarize(records)` returns one row per cell, sorted by cell name. +- Each row is `{ cell, accepted_pass, last_run }`. +- `accepted_pass` is the complete most recent `PASS` record for the cell, or + `null` when the cell has no pass. +- `last_run` is the complete last record for the cell. +- Neither exported function mutates caller-owned records. + +## Invariants and allowed actions + +Use only the repository's existing Node, Bash, and Git tools. Do not access the +network or add dependencies. Do not weaken or replace tests. No action outside +the frozen repository is authorized. + +## Evidence and done condition + +Run `npm test`. Completion additionally requires the evaluator's held-out tests +to pass and a diff containing only `src/report.mjs`. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv new file mode 100644 index 00000000..b81c6230 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv @@ -0,0 +1,3 @@ +revision parent reason body_sha256 +0000 - initial frozen repository task 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +0001 0000 human steering tightens input validation without changing scope aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md new file mode 100644 index 00000000..c12bbbca --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md @@ -0,0 +1,6 @@ +# Evaluator receipt destination + +Future B-arm runs write evaluator-owned receipts here using +`receipt-schema.tsv`. These files report observed experiment outcomes; they do +not replace the durable direct brief, grant authority, or imply that this +model-free scaffold ran an agent. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv new file mode 100644 index 00000000..13c0721c --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv @@ -0,0 +1,3 @@ +arm input_kind initial_source local_resume_source steering_source receipt_surface allowed_tools wall_budget_seconds model_token_budget judge_profile done_condition +A versioned-catalog-plan arm-a/catalog/plans/receipt-report/versions/0000.md local catalog plan version plus evaluator receipt arm-a/catalog/plans/receipt-report/versions/0001.md arm-a/catalog/plans/receipt-report/receipts node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change +B durable-direct-brief arm-b/inbox/brief-0000.md local durable brief thread plus evaluator receipt arm-b/inbox/brief-0001.md arm-b/receipts node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change diff --git a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv new file mode 100644 index 00000000..34e85253 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv @@ -0,0 +1,3 @@ +blocker_id owner dependency required_evidence status +st2-plan-runtime st2-owner experimental plans branch exact immutable source head, supported plan syntax and CLI behavior, reproducible binary path and SHA256, and focused validation receipts blocked +live-ab evals-owner future paid/live execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh new file mode 100755 index 00000000..e4963574 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="${CATALOG:?CATALOG must be set}" +experiment="$root/experiment.tsv" +arms="$root/arms.tsv" +scenario="$root/scenario.tsv" +measurements="$root/measurement-schema.tsv" +receipts="$root/receipt-schema.tsv" +blockers="$root/blockers.tsv" +lineage="$root/arm-a/catalog/plans/receipt-report/lineage.tsv" +thread="$root/arm-b/inbox/thread.tsv" +plan_0000="$root/arm-a/catalog/plans/receipt-report/versions/0000.md" +plan_0001="$root/arm-a/catalog/plans/receipt-report/versions/0001.md" +brief_0000="$root/arm-b/inbox/brief-0000.md" +brief_0001="$root/arm-b/inbox/brief-0001.md" + +expect_header() { + file="$1" + expected="$2" + test "$(head -n 1 "$file")" = "$expected" +} + +tree_hash() { + ( + cd "$1" + find . -type f -print0 | + LC_ALL=C sort -z | + xargs -0 sha256sum | + sha256sum | + awk '{print $1}' + ) +} + +expect_header "$experiment" $'field\tvalue' +expect_header "$arms" $'arm\tinput_kind\tinitial_source\tlocal_resume_source\tsteering_source\treceipt_surface\tallowed_tools\twall_budget_seconds\tmodel_token_budget\tjudge_profile\tdone_condition' +expect_header "$scenario" $'ordinal\tevent\tarm_a_input\tarm_b_input\tinvariant' +expect_header "$measurements" $'metric\ttype\tunit\trole\tpreferred_direction\tmodel_free_value\tlive_source' +expect_header "$receipts" $'field\ttype\trequired\tdescription' +expect_header "$blockers" $'blocker_id\towner\tdependency\trequired_evidence\tstatus' +expect_header "$lineage" $'version\tparent\treason\tbody_sha256' +expect_header "$thread" $'revision\tparent\treason\tbody_sha256' + +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$arms")" -eq 2 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$scenario")" -eq 6 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$measurements")" -eq 10 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$receipts")" -eq 17 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$blockers")" -eq 2 + +grep -Fqx $'allowed_tools\tnode,bash,git' "$experiment" +grep -Fqx $'network_policy\tdisabled' "$experiment" +grep -Fqx $'wall_budget_seconds\t1200' "$experiment" +grep -Fqx $'model_token_budget\t40000' "$experiment" +grep -Fqx $'initial_revision\t0000' "$experiment" +grep -Fqx $'steering_revision\t0001' "$experiment" + +cmp -s "$plan_0000" "$brief_0000" +cmp -s "$plan_0001" "$brief_0001" +test "$(sha256sum "$plan_0000" | awk '{print $1}')" != \ + "$(sha256sum "$plan_0001" | awk '{print $1}')" + +awk -F '\t' ' + NR == 2 { + tools = $7 + wall = $8 + tokens = $9 + judge = $10 + done = $11 + next + } + NR == 3 { + if ($7 != tools || $8 != wall || $9 != tokens || $10 != judge || $11 != done) { + exit 1 + } + } +' "$arms" +grep -Fqx $'A\tversioned-catalog-plan\tarm-a/catalog/plans/receipt-report/versions/0000.md\tlocal catalog plan version plus evaluator receipt\tarm-a/catalog/plans/receipt-report/versions/0001.md\tarm-a/catalog/plans/receipt-report/receipts\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" +grep -Fqx $'B\tdurable-direct-brief\tarm-b/inbox/brief-0000.md\tlocal durable brief thread plus evaluator receipt\tarm-b/inbox/brief-0001.md\tarm-b/receipts\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" + +expected_0000="$(sha256sum "$plan_0000" | awk '{print $1}')" +expected_0001="$(sha256sum "$plan_0001" | awk '{print $1}')" +grep -Fqx "0000 - initial frozen repository task $expected_0000" "$lineage" +grep -Fqx "0001 0000 human steering tightens input validation without changing scope $expected_0001" "$lineage" +grep -Fqx "0000 - initial frozen repository task $expected_0000" "$thread" +grep -Fqx "0001 0000 human steering tightens input validation without changing scope $expected_0001" "$thread" + +if ( + cd "$root/task-repo" + npm test >/dev/null 2>&1 +); then + echo "seed repository unexpectedly passes before implementation" >&2 + exit 1 +fi + +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT +cp -a "$root/task-repo" "$scratch/candidate" +cp "$root/reference/report.mjs" "$scratch/candidate/src/report.mjs" +"$root/judges/judge-task.sh" "$scratch/candidate" >/dev/null +echo "TASK-CONTRACT-GREEN-43af" + +if rg -n -i 'arm[-_ ]?[ab]|catalog[ -]?plan|direct[ -]?brief' \ + "$root/judges/judge-task.sh" "$root/judges/report.test.mjs"; then + echo "arm-neutral correctness judge contains delivery-arm identity" >&2 + exit 1 +fi +echo "ARM-PARITY-GREEN-43af" + +matrix_output="$(CATALOG="$root" "$root/run-recovery-matrix.sh")" +grep -Fqx 'RECOVERY-GREEN-A-43af' <<<"$matrix_output" +grep -Fqx 'RECOVERY-GREEN-B-43af' <<<"$matrix_output" +echo "COLD-RESTART-GREEN-43af" +echo "LOCAL-PARTITION-GREEN-43af" +echo "HUMAN-STEERING-GREEN-43af" + +test "$(awk -F '\t' 'NR > 1 && $4 == "gating" { count++ } END { print count + 0 }' "$measurements")" -eq 4 +test "$(awk -F '\t' 'NR > 1 && $4 == "reporting" { count++ } END { print count + 0 }' "$measurements")" -eq 6 +awk -F '\t' ' + NR > 1 && $4 == "gating" && $5 != "pass" { exit 1 } + NR > 1 && $4 == "reporting" && $5 != "lower" { exit 1 } +' "$measurements" +test "$(awk -F '\t' 'NR > 1 && $3 != "yes" { count++ } END { print count + 0 }' "$receipts")" -eq 0 +echo "MEASUREMENT-CONTRACT-GREEN-43af" + +checked=0 +while IFS=$'\t' read -r path expected; do + if test "$path" = "path"; then + test "$expected" = "sha256" + continue + fi + if test "$path" = "task-repo.tree"; then + test "$(tree_hash "$root/task-repo")" = "$expected" + checked=$((checked + 1)) + continue + fi + test -f "$root/$path" + actual="$(sha256sum "$root/$path" | awk '{print $1}')" + test "$actual" = "$expected" + checked=$((checked + 1)) +done <"$root/provenance.tsv" +test "$checked" -eq 23 +echo "PROVENANCE-GREEN-43af" + +test "$(awk -F '\t' 'NR > 1 && $5 == "blocked" { count++ } END { print count + 0 }' "$blockers")" -eq 2 +grep -Fq 'exact immutable source head' "$blockers" +grep -Fq 'reproducible binary path and SHA256' "$blockers" +grep -Fq 'separate authorization' "$blockers" +test -z "$(find "$root/arm-a" -type f -name '*.kdl' -print -quit)" +echo "PRODUCT-BLOCKERS-GREEN-43af" + +if rg -n --pcre2 \ + '(^|[;&|][[:space:]]*)(claude|codex|curl|wget|ssh|gh|st2[[:space:]]+(up|eval|ping)|pty[[:space:]]+(run|send)|eval)[[:space:]]' \ + "$root" -g '*.sh' -g '*.kdl'; then + echo "fixture contains a provider, network, product runtime, or nested eval command" >&2 + exit 1 +fi +echo "HERMETIC-SCAFFOLD-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv new file mode 100644 index 00000000..c53a344b --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv @@ -0,0 +1,15 @@ +field value +experiment_id catalog-plan-vs-direct-brief-v1 +source_gist https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb +task_id receipt-report-latest-pass +seed_repository task-repo +allowed_change src/report.mjs +allowed_tools node,bash,git +network_policy disabled +wall_budget_seconds 1200 +model_token_budget 40000 +judge_profile arm-neutral-public-plus-held-out +done_condition public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change +initial_revision 0000 +steering_revision 0001 +live_run_status blocked-pending-exact-st2-plan-artifact-and-separate-paid-authorization diff --git a/cells/catalog-plan-vs-direct-brief/fixture/judges/judge-task.sh b/cells/catalog-plan-vs-direct-brief/fixture/judges/judge-task.sh new file mode 100755 index 00000000..fbc47dc0 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/judges/judge-task.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +fixture_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +candidate="${1:?usage: judge-task.sh CANDIDATE_REPOSITORY}" +seed="$fixture_root/task-repo" + +test -f "$candidate/src/report.mjs" + +mapfile -t seed_files < <( + cd "$seed" + find . -type f -print | LC_ALL=C sort +) +mapfile -t candidate_files < <( + cd "$candidate" + find . -type f ! -path './.git/*' -print | LC_ALL=C sort +) +test "${seed_files[*]}" = "${candidate_files[*]}" + +while IFS= read -r relative; do + case "$relative" in + ./src/report.mjs) continue ;; + esac + cmp -s "$seed/${relative#./}" "$candidate/${relative#./}" +done < <(printf '%s\n' "${seed_files[@]}") + +( + cd "$candidate" + npm test +) +CANDIDATE_REPORT="$candidate/src/report.mjs" \ + node --test "$fixture_root/judges/report.test.mjs" + +echo "ARM-NEUTRAL-TASK-JUDGE-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/judges/report.test.mjs b/cells/catalog-plan-vs-direct-brief/fixture/judges/report.test.mjs new file mode 100644 index 00000000..c4abd76a --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/judges/report.test.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; + +const candidate = process.env.CANDIDATE_REPORT; +if (!candidate) { + throw new Error("CANDIDATE_REPORT is required"); +} + +const { parseJsonLines, summarize } = await import(pathToFileURL(candidate)); + +test("retains complete records without mutating caller input", () => { + const records = [ + { run_id: "r1", cell: "zeta", result: "PASS", usage: { input: 10 } }, + { run_id: "r2", cell: "alpha", result: "FAIL", note: "first" }, + { run_id: "r3", cell: "zeta", result: "FAIL", usage: { input: 11 } }, + { run_id: "r4", cell: "alpha", result: "PASS", note: "accepted" }, + ]; + const original = structuredClone(records); + + assert.deepEqual(summarize(records), [ + { + cell: "alpha", + accepted_pass: { + run_id: "r4", + cell: "alpha", + result: "PASS", + note: "accepted", + }, + last_run: { + run_id: "r4", + cell: "alpha", + result: "PASS", + note: "accepted", + }, + }, + { + cell: "zeta", + accepted_pass: { + run_id: "r1", + cell: "zeta", + result: "PASS", + usage: { input: 10 }, + }, + last_run: { + run_id: "r3", + cell: "zeta", + result: "FAIL", + usage: { input: 11 }, + }, + }, + ]); + assert.deepEqual(records, original); +}); + +test("rejects duplicate run ids in parsed and direct records", () => { + const duplicateText = [ + '{"run_id":"same","cell":"alpha","result":"PASS"}', + '{"run_id":"same","cell":"beta","result":"FAIL"}', + ].join("\n"); + + assert.throws( + () => parseJsonLines(duplicateText), + (error) => + error instanceof TypeError && + /duplicate/.test(error.message) && + /same/.test(error.message), + ); + assert.throws( + () => + summarize([ + { run_id: "same", cell: "alpha", result: "PASS" }, + { run_id: "same", cell: "alpha", result: "FAIL" }, + ]), + (error) => + error instanceof TypeError && + /duplicate/.test(error.message) && + /same/.test(error.message), + ); +}); + +test("rejects unknown results and identifies the run", () => { + assert.throws( + () => + parseJsonLines( + '{"run_id":"mystery-7","cell":"alpha","result":"CANCELLED"}', + ), + (error) => + error instanceof TypeError && + /unknown result/.test(error.message) && + /mystery-7/.test(error.message), + ); +}); + +test("reports malformed and structurally invalid input lines", () => { + assert.throws( + () => parseJsonLines('\n{"run_id":"ok","cell":"alpha","result":"PASS"}\n{'), + (error) => error instanceof TypeError && /line 3/.test(error.message), + ); + assert.throws( + () => parseJsonLines('["not","an","object"]'), + (error) => error instanceof TypeError && /line 1/.test(error.message), + ); + assert.throws( + () => parseJsonLines('{"run_id":"","cell":"alpha","result":"PASS"}'), + (error) => + error instanceof TypeError && + /line 1/.test(error.message) && + /run_id/.test(error.message), + ); +}); diff --git a/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv b/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv new file mode 100644 index 00000000..ff49ab3b --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv @@ -0,0 +1,11 @@ +metric type unit role preferred_direction model_free_value live_source +correctness boolean pass/fail gating pass reference-pass arm-neutral held-out judge +cold_restart_recovery boolean pass/fail gating pass pass evaluator recovery receipt +partition_recovery boolean pass/fail gating pass pass evaluator recovery receipt +human_steering boolean pass/fail gating pass pass evaluator steering receipt +coordination_messages integer count reporting lower unmeasured st2 message receipt +coordination_bytes integer bytes reporting lower unmeasured st2 message and sync receipt +model_input_tokens integer tokens reporting lower 0 harness usage receipt +model_output_tokens integer tokens reporting lower 0 harness usage receipt +cost_usd decimal USD reporting lower 0 harness usage receipt +wall_duration decimal seconds reporting lower unmeasured evaluator run receipt diff --git a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv new file mode 100644 index 00000000..583015f6 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv @@ -0,0 +1,24 @@ +path sha256 +experiment.tsv ea20f72c7221f6f2d7b41f4f2bdd1868960c0142584d76d553d172e4501fbad2 +arms.tsv 815eb439c6e4da9239e788a2de11794f81ec39d14f9e3061dc2973edd06b152f +scenario.tsv b96c25507c1866427f29e4e594d24f78455b3b5a2ab3fefa785515cc7d580677 +measurement-schema.tsv 7ce80573e9bf6957a0222e0417fc64e6592c29be9a981186327a33f6de1cbb48 +receipt-schema.tsv 954c290299434e256fd03a0eef54938b3e2940b7919e1b5604986f0baa40c904 +blockers.tsv ff046fd9eff27f5fd9ef702e8d62a0693906564f67087608746c8099516d6c50 +arm-a/catalog/plans/receipt-report/lineage.tsv fe5ef897d0f4a00172ae08ee9024af45db6f7aa6915abe9aa66f78e2620ea162 +arm-a/catalog/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +arm-a/catalog/plans/receipt-report/versions/0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 +arm-b/inbox/brief-0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +arm-b/inbox/brief-0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 +arm-b/inbox/thread.tsv 07aaee1152a5dd647cdd8cfb17e827cdc886d47936987c223a310a7f3375a86b +arm-a/catalog/plans/receipt-report/receipts/README.md be10857144f3ef9185838396e799ba1a085c7d012c01f1f61ceee5bc489b2715 +arm-b/receipts/README.md 4ad0cd92937a4a33f0202f3aaa94073d4d11a961dff3851007b4665b8e77900a +task-repo/package.json 8f756153985ab193ec21992e4adf874db4b02330db019006c5214550c2ff9618 +task-repo/README.md c50a603eac8bb3dad994dfc96f682f7f8d9a56300e21a7f7be0d040308ff580b +task-repo/src/report.mjs e2bc26ea2e19e14290585388c4f7d2cc10f56294cb4547068af884847197f020 +task-repo/test/report.test.mjs e9008004b55687df44025f66abe2e5ff739de8d8c7bec27d8d2815defd3a3a31 +reference/report.mjs 7343d91050754693f1ced37a97dae6f8701549af27ee78e08bab260a5d6550c9 +judges/report.test.mjs 24f41a2e596c940a8a3c5123e53b635bc66deb11d48bc0911a280d7a5485e670 +judges/judge-task.sh 23680852e5d3ffd744eb00b6edf12c85f5560cbe3b406da45169e710eff2a10b +run-recovery-matrix.sh 128e3caa9d125614bec70dc2ad3b7e5d1ef9f0ac442ea4dd54b9ad2f8245e5e6 +task-repo.tree 22c368e3d2b9722c0fe53710dbd16e2010e98d7215db0aa42fe7532cfa9233c5 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv b/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv new file mode 100644 index 00000000..4b1b3c78 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv @@ -0,0 +1,18 @@ +field type required description +receipt_id string yes evaluator-generated unique run receipt id +arm enum:A|B yes experiment arm +phase enum:accepted|recovered|steered|completed yes observed evaluator phase +intent_revision string yes exact accepted intent revision +intent_sha256 sha256 yes exact intent body hash +seed_repo_sha256 sha256 yes frozen repository tree hash +correctness enum:pass|fail|not-run yes arm-neutral judge outcome +cold_restart_recovery enum:pass|fail|not-run yes exact local intent recovery outcome +partition_recovery enum:pass|fail|not-run yes recovery while remote input is unavailable +human_steering enum:pass|fail|not-run yes revised intent acceptance outcome +coordination_messages nonnegative-integer yes observed external message count +coordination_bytes nonnegative-integer yes observed message and sync payload bytes +model_input_tokens nonnegative-integer yes harness-reported input tokens +model_output_tokens nonnegative-integer yes harness-reported output tokens +cost_usd nonnegative-decimal yes harness-reported cost +wall_duration_seconds nonnegative-decimal yes evaluator elapsed wall time +evidence string yes path or immutable URL for supporting output diff --git a/cells/catalog-plan-vs-direct-brief/fixture/reference/report.mjs b/cells/catalog-plan-vs-direct-brief/fixture/reference/report.mjs new file mode 100644 index 00000000..6b8f7441 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/reference/report.mjs @@ -0,0 +1,76 @@ +function validateRecord(record, context) { + if (record === null || typeof record !== "object" || Array.isArray(record)) { + throw new TypeError(`${context}: expected an object record`); + } + + for (const field of ["run_id", "cell", "result"]) { + if (typeof record[field] !== "string" || record[field].length === 0) { + throw new TypeError(`${context}: missing non-empty ${field}`); + } + } + + if (record.result !== "PASS" && record.result !== "FAIL") { + throw new TypeError( + `${context}: unknown result ${record.result} for ${record.run_id}`, + ); + } +} + +function validateUnique(records, contextForIndex) { + const seen = new Set(); + records.forEach((record, index) => { + const context = contextForIndex(index); + validateRecord(record, context); + if (seen.has(record.run_id)) { + throw new TypeError(`${context}: duplicate run_id ${record.run_id}`); + } + seen.add(record.run_id); + }); +} + +export function parseJsonLines(text) { + const records = []; + const sourceLines = text.split("\n"); + + sourceLines.forEach((line, index) => { + if (line.trim().length === 0) { + return; + } + + let record; + try { + record = JSON.parse(line); + } catch { + throw new TypeError(`line ${index + 1}: malformed JSON`); + } + records.push({ record, lineNumber: index + 1 }); + }); + + validateUnique( + records.map(({ record }) => record), + (index) => `line ${records[index].lineNumber}`, + ); + return records.map(({ record }) => record); +} + +export function summarize(records) { + validateUnique(records, (index) => `record ${index + 1}`); + + const byCell = new Map(); + for (const record of records) { + const current = byCell.get(record.cell) ?? { + cell: record.cell, + accepted_pass: null, + last_run: null, + }; + if (record.result === "PASS") { + current.accepted_pass = { ...record }; + } + current.last_run = { ...record }; + byCell.set(record.cell, current); + } + + return [...byCell.values()].sort((left, right) => + left.cell.localeCompare(right.cell), + ); +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh b/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh new file mode 100755 index 00000000..20ecd1f4 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="${CATALOG:?CATALOG must be set}" +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +tree_hash() { + ( + cd "$1" + find . -type f -print0 | + LC_ALL=C sort -z | + xargs -0 sha256sum | + sha256sum | + awk '{print $1}' + ) +} + +seed_hash="$(tree_hash "$root/task-repo")" +receipt_header=$'receipt_id\tarm\tphase\tintent_revision\tintent_sha256\tseed_repo_sha256\tcorrectness\tcold_restart_recovery\tpartition_recovery\thuman_steering\tcoordination_messages\tcoordination_bytes\tmodel_input_tokens\tmodel_output_tokens\tcost_usd\twall_duration_seconds\tevidence' + +run_arm() { + arm="$1" + remote="$scratch/$arm/remote" + local_store="$scratch/$arm/local" + session="$scratch/$arm/session" + receipts="$scratch/$arm/receipts.tsv" + + mkdir -p "$remote" "$local_store" "$session" + printf '%s\n' "$receipt_header" >"$receipts" + + case "$arm" in + A) + cp -a "$root/arm-a/catalog" "$remote/catalog" + initial_source="$remote/catalog/plans/receipt-report/versions/0000.md" + steered_source="$remote/catalog/plans/receipt-report/versions/0001.md" + initial_local="$local_store/catalog/plans/receipt-report/versions/0000.md" + steered_local="$local_store/catalog/plans/receipt-report/versions/0001.md" + mkdir -p "$(dirname "$initial_local")" + ;; + B) + cp -a "$root/arm-b/inbox" "$remote/inbox" + initial_source="$remote/inbox/brief-0000.md" + steered_source="$remote/inbox/brief-0001.md" + initial_local="$local_store/inbox/brief-0000.md" + steered_local="$local_store/inbox/brief-0001.md" + mkdir -p "$(dirname "$initial_local")" + ;; + *) + echo "unknown arm: $arm" >&2 + return 2 + ;; + esac + + initial_hash="$(sha256sum "$initial_source" | awk '{print $1}')" + steered_hash="$(sha256sum "$steered_source" | awk '{print $1}')" + cp "$initial_source" "$initial_local" + printf '%s\n' \ + "${arm}-accepted-0000 ${arm} accepted 0000 ${initial_hash} ${seed_hash} not-run not-run not-run not-run 0 0 0 0 0 0 local:${initial_local#$scratch/}" \ + >>"$receipts" + + printf 'volatile state\n' >"$session/state" + rm -rf -- "$session" + mv "$remote" "$remote.offline" + test ! -e "$remote" + test "$(sha256sum "$initial_local" | awk '{print $1}')" = "$initial_hash" + printf '%s\n' \ + "${arm}-recovered-0000 ${arm} recovered 0000 ${initial_hash} ${seed_hash} not-run pass pass not-run 0 0 0 0 0 0 local:${initial_local#$scratch/}" \ + >>"$receipts" + + mv "$remote.offline" "$remote" + cp "$steered_source" "$steered_local" + printf '%s\n' \ + "${arm}-steered-0001 ${arm} steered 0001 ${steered_hash} ${seed_hash} not-run pass not-run pass 0 0 0 0 0 0 local:${steered_local#$scratch/}" \ + >>"$receipts" + + mkdir -p "$session" + printf 'volatile state\n' >"$session/state" + rm -rf -- "$session" + mv "$remote" "$remote.offline" + test ! -e "$remote" + test "$(sha256sum "$steered_local" | awk '{print $1}')" = "$steered_hash" + test "$(sha256sum "$initial_local" | awk '{print $1}')" = "$initial_hash" + printf '%s\n' \ + "${arm}-recovered-0001 ${arm} recovered 0001 ${steered_hash} ${seed_hash} not-run pass pass pass 0 0 0 0 0 0 local:${steered_local#$scratch/}" \ + >>"$receipts" + + awk -F '\t' ' + NR == 1 { expected = 17 } + NF != expected { exit 1 } + NR > 1 && $2 != "'"$arm"'" { exit 1 } + END { if (NR != 5) exit 1 } + ' "$receipts" + echo "RECOVERY-GREEN-${arm}-43af" +} + +run_arm A +run_arm B diff --git a/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv b/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv new file mode 100644 index 00000000..021b7bf0 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv @@ -0,0 +1,7 @@ +ordinal event arm_a_input arm_b_input invariant +1 initial_delivery versions/0000.md brief-0000.md byte-identical intent and one durable acceptance receipt +2 cold_restart local version 0000 local brief 0000 no volatile conversation or process state +3 simulated_partition local catalog only local durable inbox only remote input source unavailable during recovery +4 human_steering versions/0001.md brief-0001.md byte-identical complete revised intent +5 restart_after_steering local versions 0000 and 0001 local briefs 0000 and 0001 revision 0001 selected and revision 0000 retained +6 future_completion same arm-neutral judge same arm-neutral judge report correctness recovery traffic and cost without preferred-arm thresholds diff --git a/cells/catalog-plan-vs-direct-brief/fixture/task-repo/README.md b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/README.md new file mode 100644 index 00000000..093850f7 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/README.md @@ -0,0 +1,11 @@ +# receipt-report + +A dependency-free report core for append-only eval run receipts. + +Run the visible test suite with: + +```sh +npm test +``` + +The task brief is delivered separately. Only `src/report.mjs` is in scope. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/task-repo/package.json b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/package.json new file mode 100644 index 00000000..a34c7461 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/package.json @@ -0,0 +1,9 @@ +{ + "name": "receipt-report", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "test": "node --test" + } +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/task-repo/src/report.mjs b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/src/report.mjs new file mode 100644 index 00000000..7b5ae6e4 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/src/report.mjs @@ -0,0 +1,13 @@ +export function parseJsonLines(text) { + return text + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +export function summarize(records) { + return records.map((record) => ({ + cell: record.cell, + last_run: record.result, + })); +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/task-repo/test/report.test.mjs b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/test/report.test.mjs new file mode 100644 index 00000000..f3cacc1a --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/task-repo/test/report.test.mjs @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { parseJsonLines, summarize } from "../src/report.mjs"; + +test("reports the latest accepted pass separately from the last run", () => { + const records = parseJsonLines(` +{"run_id":"r1","cell":"alpha","result":"PASS","score":"2/2"} +{"run_id":"r2","cell":"alpha","result":"FAIL","score":"1/2"} +{"run_id":"r3","cell":"beta","result":"FAIL","score":"0/1"} +`); + + assert.deepEqual(summarize(records), [ + { + cell: "alpha", + accepted_pass: { + run_id: "r1", + cell: "alpha", + result: "PASS", + score: "2/2", + }, + last_run: { + run_id: "r2", + cell: "alpha", + result: "FAIL", + score: "1/2", + }, + }, + { + cell: "beta", + accepted_pass: null, + last_run: { + run_id: "r3", + cell: "beta", + result: "FAIL", + score: "0/1", + }, + }, + ]); +}); + +test("ignores blank lines and sorts cells", () => { + const records = parseJsonLines(` + +{"run_id":"r1","cell":"zeta","result":"PASS"} + +{"run_id":"r2","cell":"alpha","result":"PASS"} +`); + + assert.deepEqual( + summarize(records).map(({ cell }) => cell), + ["alpha", "zeta"], + ); +}); diff --git a/evidence/harness-exclusions.tsv b/evidence/harness-exclusions.tsv index fb928d87..ae5c7a30 100644 --- a/evidence/harness-exclusions.tsv +++ b/evidence/harness-exclusions.tsv @@ -1,4 +1,5 @@ cell surface reason +catalog-plan-vs-direct-brief cell deterministic plan-versus-brief fixture and recovery contract; no harness seat context-resource-continuity cell deterministic native context/resource restart probe; no harness seat docs judge:cold-reader one-shot offline Claude print grader; no bus identity, DING, or hook surface exec-lifecycle-logging cell deterministic native exec lifecycle probe; no harness seat From 43247a2de2fe4c39ad41b0170fe63bea78d6ebd9 Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 31 Jul 2026 01:21:21 +0200 Subject: [PATCH 2/6] Pair catalog plan eval with st2 inspector --- AGENT-SPEC.md | 69 +++++++++++- CATALOG.md | 2 +- README.md | 4 +- bin/check-corpus.sh | 20 ++-- bin/check-preflight-safety.sh | 3 + bin/check-st2-pin-consistency.sh | 24 ++++ bin/st2-pin.sh | 6 + cells/catalog-plan-vs-direct-brief/README.md | 24 ++-- .../catalog-plan-vs-direct-brief.kdl | 27 ++++- .../agents/eval/receipt-worker/agent.kdl | 5 + .../catalog/plans/receipt-report/plan.kdl | 9 ++ .../fixture/blockers.tsv | 2 +- .../fixture/check-fixtures.sh | 42 +++++-- .../fixture/check-plan-contract.sh | 104 ++++++++++++++++++ .../fixture/experiment.tsv | 8 +- .../fixture/invalid/current/0000.md | 1 + .../fixture/invalid/current/plan.kdl | 5 + .../fixture/provenance.tsv | 10 +- .../fixture/runner.tsv | 9 ++ 19 files changed, 329 insertions(+), 45 deletions(-) create mode 100755 bin/check-st2-pin-consistency.sh create mode 100755 bin/st2-pin.sh create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl create mode 100755 cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/invalid/current/0000.md create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/runner.tsv diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index f1e46481..28ccdcd1 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -1,8 +1,8 @@ # Canonical st2 agent specification This is the sole agent-authoring specification for this repository. It is pinned to st2 -[`9887b2842222def0838c2cd82e6c24c218f7efa6`](https://github.com/compoundingtech/st2/commit/9887b2842222def0838c2cd82e6c24c218f7efa6) -(`0.1.0`, source `9887b28`). It documents the hand-authored KDL accepted at that commit. Do not infer +[`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`](https://github.com/compoundingtech/st2/commit/2caa0d7f159c3c0d9c483bd63b2579d33f1986ff) +(`0.1.0`, source `2caa0d7`). It documents the hand-authored KDL accepted at that commit. Do not infer additional fields or commands from older corpus fixtures. st2 runs long-lived `service` agents made of interactive `pty` tasks and terminal-free `exec` tasks. @@ -201,6 +201,64 @@ task named `agent`. Network startup materializes active declarations for the sel reconciliation; retired and other-host declarations are skipped. A gating failure suppresses only that agent. Prefer a catalog-owned `.st2/` overlay and locally excluded tool loaders. +## Experimental read-only plans + +The experimental plan surface is pinned to the +[source sketch revision `5c1d142`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994). +An external plan is a top-level declaration: + +```kdl +plan "ship-remote-approvals" { + owner "app-web" + version "0000" resource="file:versions/0000.md" + version "0001" resource="file:versions/0001.md" { + parent "0000" + why "Browser proof exposed an approval race." + } +} +``` + +An agent links it with a source-relative declaration: + +```kdl +agent "app-web" { + plan-ref "file:plans/ship-remote-approvals/plan.kdl" +} +``` + +The equivalent agent-local form derives its owner from the containing explicit +agent identity: + +```kdl +agent "app-web" { + plan "review-follow-up" { + version "0000" resource="file:plans/review-follow-up.md" + } +} +``` + +Plan identity is the explicit KDL value, never its directory. External plans +require one `owner`. Each version requires one source-relative `file:` resource. +A version may repeat `parent` for multiple immutable parents; parents must +exist, be unique, and form an acyclic graph. A version with parents requires +one non-empty `why`. The sorted frontier is derived as every version with no +child, retaining concurrent siblings. `plan-ref` and version resources resolve +relative to their declaring KDL and must remain inside the selected catalog. + +The only supported CLI is read-only: + +```sh +st2 plan validate [PATH] [--json] +st2 plan list [PATH] [--json] +st2 plan show [PATH] [--json] +st2 plan inspect [PATH] [--json] +``` + +Global `--catalog` selects the same input. The experiment has no current +pointer, execution, controller, schedule, step graph, retry, claim, receipt, +event, reconciliation mutation, agent restart, CAS, merge, or deployment +behavior. Direct KDL and direct human-to-agent planning remain supported. + ## Validation, health, and lifecycle Canonical validation is: @@ -432,8 +490,7 @@ Inspect the declaration, every referenced template, and every workspace destinat materialization command. Materialization is byte-idempotent and does not imply hook installation. Starting the network is a separate, explicitly authorized action. -For source `9887b28`, the accepted Linux executable has SHA256 -`d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d`; its published archive has SHA256 -`32ee103bd17ccb3e155ac63d816a3906c2470a3c98e3cc04b56e5a67138b9927`. `bin/check-corpus.sh` verifies -the variable-age version contract, exact installed binary, embedded full source commit, strict semantic +For source `2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`, the accepted local-source Linux executable has +SHA256 `83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38`. `bin/check-corpus.sh` verifies +the variable-age version contract, exact installed binary, shared source-pin consistency, strict semantic validation, fixture resets, and the rest of the model-free corpus gate before an eval may run. diff --git a/CATALOG.md b/CATALOG.md index 702e7867..28ac428e 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -16,7 +16,7 @@ harness-native loader plus canonical hook file. | Cell | Harness | Model(s) / effort | Model seats | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | |---|---|---|---:|---|---|---:|---|---| -| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 9 | — | **NO STRUCTURED RUN** | +| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 15 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | diff --git a/README.md b/README.md index 37808f25..c1a9e9d1 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,9 @@ VERDICT: PASS ``` Requirements are `st2 0.1.0` from source -[`9887b28`](https://github.com/compoundingtech/st2/commit/9887b2842222def0838c2cd82e6c24c218f7efa6), +[`2caa0d7`](https://github.com/compoundingtech/st2/commit/2caa0d7f159c3c0d9c483bd63b2579d33f1986ff), +using the accepted local-source Linux binary SHA256 +`83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38`; `pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and Node for JavaScript fixtures. A paid cell also needs every harness named by its dry-run row. diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index fc85e69d..5bde6c79 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -5,26 +5,20 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -expected_source="9887b28" -expected_source_full="9887b2842222def0838c2cd82e6c24c218f7efa6" -expected_binary_sha256="d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d" -expected_version_regex='^st2 0\.1\.0 — running from local source \(9887b28, .+ ago\)$' +source bin/st2-pin.sh +bin/check-st2-pin-consistency.sh st2_path="$(command -v st2)" actual_version="$(st2 --version)" -[[ "$actual_version" =~ $expected_version_regex ]] || { - echo "FAIL: expected st2 0.1.0 from pinned source $expected_source, found $actual_version" >&2 +[[ "$actual_version" =~ $ST2_VERSION_REGEX ]] || { + echo "FAIL: expected st2 0.1.0 from pinned source $ST2_SOURCE_SHORT, found $actual_version" >&2 exit 1 } actual_binary_sha256="$(sha256sum "$st2_path" | awk '{ print $1 }')" -[ "$actual_binary_sha256" = "$expected_binary_sha256" ] || { - echo "FAIL: expected st2 binary sha256 $expected_binary_sha256, found $actual_binary_sha256 at $st2_path" >&2 +[ "$actual_binary_sha256" = "$ST2_BINARY_SHA256" ] || { + echo "FAIL: expected st2 binary sha256 $ST2_BINARY_SHA256, found $actual_binary_sha256 at $st2_path" >&2 exit 1 } -LC_ALL=C grep -aFq "$expected_source_full" "$st2_path" || { - echo "FAIL: st2 binary at $st2_path does not embed full pinned source $expected_source_full" >&2 - exit 1 -} -echo "PASS: pinned runner source $expected_source ($actual_version; sha256 $actual_binary_sha256)" +echo "PASS: pinned runner source $ST2_SOURCE_FULL ($actual_version; sha256 $actual_binary_sha256)" mapfile -d '' shell_files < <( find bin cells -type f -name '*.sh' -not -path '*/_git/*' -print0 | sort -z diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index 4af8f41e..978b114f 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -20,6 +20,7 @@ reachable=( bin/check-preflight-safety.sh bin/check-run-history.sh bin/check-retired-surfaces.sh + bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh @@ -27,6 +28,7 @@ reachable=( bin/corpus-inventory.sh bin/generate-catalog.sh bin/model-seat-inventory.sh + bin/st2-pin.sh ) materializers=( cells/signal-rename/fixture/materialize.sh @@ -68,6 +70,7 @@ expected_direct=( bin/check-preflight-safety.sh bin/check-retired-surfaces.sh bin/check-run-history.sh + bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh diff --git a/bin/check-st2-pin-consistency.sh b/bin/check-st2-pin-consistency.sh new file mode 100755 index 00000000..82c5cff9 --- /dev/null +++ b/bin/check-st2-pin-consistency.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +source bin/st2-pin.sh + +grep -Fq "$ST2_SOURCE_FULL" AGENT-SPEC.md +grep -Fq "$ST2_SOURCE_FULL" README.md +grep -Fq "$ST2_BINARY_SHA256" AGENT-SPEC.md +grep -Fq "$ST2_BINARY_SHA256" README.md +grep -Fq 'source bin/st2-pin.sh' bin/check-corpus.sh +grep -Fqx "source_full $ST2_SOURCE_FULL" cells/catalog-plan-vs-direct-brief/fixture/runner.tsv +grep -Fqx "source_short $ST2_SOURCE_SHORT" cells/catalog-plan-vs-direct-brief/fixture/runner.tsv +grep -Fqx "binary_sha256 $ST2_BINARY_SHA256" cells/catalog-plan-vs-direct-brief/fixture/runner.tsv + +if rg -n \ + '9887b28|c6846f6239329f0803142afc06c15a07b93937c1|d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d|2bba8d58be24250bc262f75f835ce2d780369add275774f3f2135c623d23d29c' \ + AGENT-SPEC.md README.md bin/check-corpus.sh bin/st2-pin.sh; then + echo "FAIL: active st2 pin surfaces retain a superseded source or binary hash" >&2 + exit 1 +fi + +echo "PASS: active spec, README, and executable preflight share st2 $ST2_SOURCE_FULL / $ST2_BINARY_SHA256" diff --git a/bin/st2-pin.sh b/bin/st2-pin.sh new file mode 100755 index 00000000..f4a2b184 --- /dev/null +++ b/bin/st2-pin.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +ST2_SOURCE_SHORT="2caa0d7" +ST2_SOURCE_FULL="2caa0d7f159c3c0d9c483bd63b2579d33f1986ff" +ST2_BINARY_SHA256="83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38" +ST2_VERSION_REGEX='^st2 0\.1\.0 — running from local source \(2caa0d7, .+ ago\)$' diff --git a/cells/catalog-plan-vs-direct-brief/README.md b/cells/catalog-plan-vs-direct-brief/README.md index ccf90135..cc8ba492 100644 --- a/cells/catalog-plan-vs-direct-brief/README.md +++ b/cells/catalog-plan-vs-direct-brief/README.md @@ -1,7 +1,7 @@ # catalog-plan-vs-direct-brief Model-free contract for the first experiment proposed by -[`st2 plans: today → dream`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb): +[`st2 plans: today → dream`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994): compare one realistic repository task delivered as a versioned local catalog plan (A) with the same task delivered as an ordinary durable direct brief (B). @@ -22,11 +22,17 @@ The deterministic fixture proves: coordination traffic and model cost remain reported comparison metrics; and - no model or provider is launched. -Arm A reserves a catalog-local plan directory with immutable Markdown versions, -lineage metadata, and receipt space. Arm B reserves equivalent durable -message-thread metadata and receipt space. The scaffold deliberately does not -invent `plan.kdl` syntax or st2 runtime behavior. Those assertions remain -blocked until the st2 owner supplies one exact experimental source head, -supported interface, reproducible artifact hash, and focused receipts. A -paid/live A/B is a separate authorization with an exact model, effort, budget, -run order, and rollback. +Arm A uses the exact external `plan.kdl` and agent `plan-ref` contract from +[st2 draft PR #115](https://github.com/compoundingtech/st2/pull/115) at source +`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`, paired to the source gist at +revision `5c1d1427c0556d95d13890e5c5086cd85b25d994`. The accepted Linux +artifact SHA256 is +`83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38`. +The fixture exercises only `plan validate`, `list`, `show`, and `inspect`, and +proves they do not alter the catalog. + +Arm B retains equivalent durable message-thread metadata and receipt space. +The product experiment adds no current pointer, execution, scheduling, steps, +retries, progress claims, events, reconciliation, or CAS. A paid/live A/B +remains a separate authorization with an exact model, effort, budget, run +order, and rollback. diff --git a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl index f1b16695..9832e459 100644 --- a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl +++ b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl @@ -6,6 +6,9 @@ eval { run "fixture-contract" { command "bash ./check-fixtures.sh" } + run "plan-contract" { + command "bash ./check-plan-contract.sh" + } judges { judge "TASK — frozen repository starts red and its arm-neutral reference implementation passes" { @@ -29,10 +32,28 @@ eval { judge "PROVENANCE — every authoritative input has a verified content hash" { exec "grep -Fqx PROVENANCE-GREEN-43af $RUNS_DIR/fixture-contract.out" } - judge "BOUNDARY — unsupported plan runtime and paid execution remain explicit blockers" { - exec "grep -Fqx PRODUCT-BLOCKERS-GREEN-43af $RUNS_DIR/fixture-contract.out" + judge "PAIRING — exact experimental source and artifact are frozen while live execution stays blocked" { + exec "grep -Fqx PRODUCT-PAIRING-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "RUNNER — the exact experimental st2 binary matches the accepted source and SHA256" { + exec "grep -Fqx PLAN-RUNNER-PIN-GREEN-43af $RUNS_DIR/plan-contract.out" + } + judge "PLAN VALIDATE — the external plan and agent reference normalize without errors" { + exec "grep -Fqx PLAN-VALIDATE-GREEN-43af $RUNS_DIR/plan-contract.out" + } + judge "PLAN LIST — explicit identity, owner, and derived frontier are exact" { + exec "grep -Fqx PLAN-LIST-GREEN-43af $RUNS_DIR/plan-contract.out" + } + judge "PLAN SHOW — normalized intent exposes versions but not provenance paths" { + exec "grep -Fqx PLAN-SHOW-GREEN-43af $RUNS_DIR/plan-contract.out" + } + judge "PLAN INSPECT — provenance and resolved resources are exact and read-only" { + exec "grep -Fqx PLAN-INSPECT-READONLY-GREEN-43af $RUNS_DIR/plan-contract.out" + } + judge "PLAN BOUNDARY — a mutable current pointer is rejected with a classified error" { + exec "grep -Fqx PLAN-BOUNDARY-GREEN-43af $RUNS_DIR/plan-contract.out" } - judge "HERMETIC — the scaffold launches no model, provider, network client, or product runtime" { + judge "HERMETIC — the scaffold launches no model, provider, network client, or mutating product runtime" { exec "grep -Fqx HERMETIC-SCAFFOLD-GREEN-43af $RUNS_DIR/fixture-contract.out" } } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl new file mode 100644 index 00000000..dacd0fe6 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl @@ -0,0 +1,5 @@ +agent "receipt-worker" { + host "eval" + command "true" + plan-ref "file:../../../plans/receipt-report/plan.kdl" +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl new file mode 100644 index 00000000..7f7f5931 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl @@ -0,0 +1,9 @@ +plan "receipt-report" { + owner "receipt-worker" + + version "0000" resource="file:versions/0000.md" + version "0001" resource="file:versions/0001.md" { + parent "0000" + why "Human steering tightens input validation without changing scope." + } +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv index 34e85253..2aa5e93e 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv @@ -1,3 +1,3 @@ blocker_id owner dependency required_evidence status -st2-plan-runtime st2-owner experimental plans branch exact immutable source head, supported plan syntax and CLI behavior, reproducible binary path and SHA256, and focused validation receipts blocked +st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact source 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff and binary SHA256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 with focused model 3/3, CLI 2/2, full workspace tests and doctests, scoped Clippy, rustfmt, diff receipts, and hosted run 30589548410 green ready live-ab evals-owner future paid/live execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh index e4963574..5ee3a452 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh @@ -8,8 +8,11 @@ scenario="$root/scenario.tsv" measurements="$root/measurement-schema.tsv" receipts="$root/receipt-schema.tsv" blockers="$root/blockers.tsv" +runner="$root/runner.tsv" lineage="$root/arm-a/catalog/plans/receipt-report/lineage.tsv" thread="$root/arm-b/inbox/thread.tsv" +plan_declaration="$root/arm-a/catalog/plans/receipt-report/plan.kdl" +agent_declaration="$root/arm-a/catalog/agents/eval/receipt-worker/agent.kdl" plan_0000="$root/arm-a/catalog/plans/receipt-report/versions/0000.md" plan_0001="$root/arm-a/catalog/plans/receipt-report/versions/0001.md" brief_0000="$root/arm-b/inbox/brief-0000.md" @@ -38,6 +41,7 @@ expect_header "$scenario" $'ordinal\tevent\tarm_a_input\tarm_b_input\tinvariant' expect_header "$measurements" $'metric\ttype\tunit\trole\tpreferred_direction\tmodel_free_value\tlive_source' expect_header "$receipts" $'field\ttype\trequired\tdescription' expect_header "$blockers" $'blocker_id\towner\tdependency\trequired_evidence\tstatus' +expect_header "$runner" $'field\tvalue' expect_header "$lineage" $'version\tparent\treason\tbody_sha256' expect_header "$thread" $'revision\tparent\treason\tbody_sha256' @@ -46,6 +50,7 @@ test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$scenario")" - test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$measurements")" -eq 10 test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$receipts")" -eq 17 test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$blockers")" -eq 2 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$runner")" -eq 8 grep -Fqx $'allowed_tools\tnode,bash,git' "$experiment" grep -Fqx $'network_policy\tdisabled' "$experiment" @@ -53,6 +58,19 @@ grep -Fqx $'wall_budget_seconds\t1200' "$experiment" grep -Fqx $'model_token_budget\t40000' "$experiment" grep -Fqx $'initial_revision\t0000' "$experiment" grep -Fqx $'steering_revision\t0001' "$experiment" +grep -Fqx $'source_gist\thttps://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" +grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" +grep -Fqx $'st2_plan_source\t2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$experiment" +grep -Fqx $'st2_plan_binary_sha256\t83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$experiment" +grep -Fqx $'live_run_status\tblocked-pending-separate-paid-authorization' "$experiment" +grep -Fqx $'st2_pr\thttps://github.com/compoundingtech/st2/pull/115' "$runner" +grep -Fqx $'source_full\t2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$runner" +grep -Fqx $'source_short\t2caa0d7' "$runner" +grep -Fqx $'binary_sha256\t83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$runner" +grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$runner" +grep -Fqx $'runtime_scope\tread-only-validate-list-show-inspect' "$runner" +grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30589548410' "$runner" +grep -Fqx $'hosted_status\tpass' "$runner" cmp -s "$plan_0000" "$brief_0000" cmp -s "$plan_0001" "$brief_0001" @@ -83,6 +101,14 @@ grep -Fqx "0000 - initial frozen repository task $expected_0000" "$lineage" grep -Fqx "0001 0000 human steering tightens input validation without changing scope $expected_0001" "$lineage" grep -Fqx "0000 - initial frozen repository task $expected_0000" "$thread" grep -Fqx "0001 0000 human steering tightens input validation without changing scope $expected_0001" "$thread" +grep -Fqx 'plan "receipt-report" {' "$plan_declaration" +grep -Fqx ' owner "receipt-worker"' "$plan_declaration" +grep -Fqx ' version "0000" resource="file:versions/0000.md"' "$plan_declaration" +grep -Fqx ' version "0001" resource="file:versions/0001.md" {' "$plan_declaration" +grep -Fqx ' parent "0000"' "$plan_declaration" +grep -Fqx ' why "Human steering tightens input validation without changing scope."' "$plan_declaration" +grep -Fqx 'agent "receipt-worker" {' "$agent_declaration" +grep -Fqx ' plan-ref "file:../../../plans/receipt-report/plan.kdl"' "$agent_declaration" if ( cd "$root/task-repo" @@ -141,20 +167,22 @@ while IFS=$'\t' read -r path expected; do test "$actual" = "$expected" checked=$((checked + 1)) done <"$root/provenance.tsv" -test "$checked" -eq 23 +test "$checked" -eq 29 echo "PROVENANCE-GREEN-43af" -test "$(awk -F '\t' 'NR > 1 && $5 == "blocked" { count++ } END { print count + 0 }' "$blockers")" -eq 2 -grep -Fq 'exact immutable source head' "$blockers" -grep -Fq 'reproducible binary path and SHA256' "$blockers" +test "$(awk -F '\t' 'NR > 1 && $5 == "ready" { count++ } END { print count + 0 }' "$blockers")" -eq 1 +test "$(awk -F '\t' 'NR > 1 && $5 == "blocked" { count++ } END { print count + 0 }' "$blockers")" -eq 1 +grep -Fq '2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$blockers" +grep -Fq '83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$blockers" +grep -Fq 'hosted run 30589548410 green' "$blockers" grep -Fq 'separate authorization' "$blockers" -test -z "$(find "$root/arm-a" -type f -name '*.kdl' -print -quit)" -echo "PRODUCT-BLOCKERS-GREEN-43af" +test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 2 +echo "PRODUCT-PAIRING-GREEN-43af" if rg -n --pcre2 \ '(^|[;&|][[:space:]]*)(claude|codex|curl|wget|ssh|gh|st2[[:space:]]+(up|eval|ping)|pty[[:space:]]+(run|send)|eval)[[:space:]]' \ "$root" -g '*.sh' -g '*.kdl'; then - echo "fixture contains a provider, network, product runtime, or nested eval command" >&2 + echo "fixture contains a provider, network, mutating product runtime, or nested eval command" >&2 exit 1 fi echo "HERMETIC-SCAFFOLD-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh new file mode 100755 index 00000000..5a115afd --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="${CATALOG:?CATALOG must be set}" +runner="$root/runner.tsv" +catalog="$root/arm-a/catalog" +invalid="$root/invalid/current" +st2_path="$(command -v st2)" + +value() { + awk -F '\t' -v key="$1" '$1 == key { print $2 }' "$runner" +} + +source_short="$(value source_short)" +expected_sha="$(value binary_sha256)" +version_regex="^st2 0\\.1\\.0 — running from local source \\(${source_short}, .+ ago\\)$" +actual_version="$("$st2_path" --version)" +actual_sha="$(sha256sum "$st2_path" | awk '{print $1}')" +[[ "$actual_version" =~ $version_regex ]] +test "$actual_sha" = "$expected_sha" +echo "PLAN-RUNNER-PIN-GREEN-43af" + +before="$(mktemp)" +after="$(mktemp)" +invalid_json="$(mktemp)" +cleanup() { + rm -f -- "$before" "$after" "$invalid_json" +} +trap cleanup EXIT + +find "$catalog" -type f -print0 | + LC_ALL=C sort -z | + xargs -0 sha256sum >"$before" + +validate_json="$("$st2_path" plan validate --catalog "$catalog" --json)" +jq -e ' + .result == "valid" and + .plans == 1 and + .errors == 0 +' <<<"$validate_json" >/dev/null +echo "PLAN-VALIDATE-GREEN-43af" + +list_json="$("$st2_path" plan list --catalog "$catalog" --json)" +jq -e ' + length == 1 and + .[0].identity == "receipt-report" and + .[0].owner == "receipt-worker" and + .[0].frontier == ["0001"] +' <<<"$list_json" >/dev/null +echo "PLAN-LIST-GREEN-43af" + +show_json="$("$st2_path" plan show receipt-report --catalog "$catalog" --json)" +jq -e ' + .identity == "receipt-report" and + .owner == "receipt-worker" and + .frontier == ["0001"] and + (.versions | length) == 2 and + .versions[0] == { + "identity": "0000", + "parents": [], + "why": null, + "resource": "file:versions/0000.md" + } and + .versions[1] == { + "identity": "0001", + "parents": ["0000"], + "why": "Human steering tightens input validation without changing scope.", + "resource": "file:versions/0001.md" + } and + (has("source") | not) and + (.versions[0] | has("resolvedResource") | not) +' <<<"$show_json" >/dev/null +echo "PLAN-SHOW-GREEN-43af" + +inspect_json="$("$st2_path" plan inspect receipt-report --catalog "$catalog" --json)" +jq -e ' + .identity == "receipt-report" and + .owner == "receipt-worker" and + .sourceKind == "external" and + .referencedBy == ["receipt-worker"] and + .frontier == ["0001"] and + (.source | endswith("/plans/receipt-report/plan.kdl")) and + (.versions[0].resolvedResource | endswith("/plans/receipt-report/versions/0000.md")) and + (.versions[1].resolvedResource | endswith("/plans/receipt-report/versions/0001.md")) +' <<<"$inspect_json" >/dev/null + +find "$catalog" -type f -print0 | + LC_ALL=C sort -z | + xargs -0 sha256sum >"$after" +cmp -s "$before" "$after" +test ! -e "$catalog/.st2" +echo "PLAN-INSPECT-READONLY-GREEN-43af" + +if "$st2_path" plan validate --catalog "$invalid" --json >"$invalid_json" 2>/dev/null; then + echo "unsupported current pointer unexpectedly validated" >&2 + exit 1 +fi +jq -e ' + .result == "invalid" and + .code == "unsupported-plan-field" and + (.error | contains("unsupported-current")) and + (.error | contains("current")) +' "$invalid_json" >/dev/null +echo "PLAN-BOUNDARY-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv index c53a344b..906ccddc 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv @@ -1,6 +1,10 @@ field value experiment_id catalog-plan-vs-direct-brief-v1 -source_gist https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb +source_gist https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994 +source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 +st2_plan_pr https://github.com/compoundingtech/st2/pull/115 +st2_plan_source 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff +st2_plan_binary_sha256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 task_id receipt-report-latest-pass seed_repository task-repo allowed_change src/report.mjs @@ -12,4 +16,4 @@ judge_profile arm-neutral-public-plus-held-out done_condition public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change initial_revision 0000 steering_revision 0001 -live_run_status blocked-pending-exact-st2-plan-artifact-and-separate-paid-authorization +live_run_status blocked-pending-separate-paid-authorization diff --git a/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/0000.md b/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/0000.md new file mode 100644 index 00000000..ef57ad9c --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/0000.md @@ -0,0 +1 @@ +# Unsupported mutable current pointer diff --git a/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl new file mode 100644 index 00000000..8b31bac7 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl @@ -0,0 +1,5 @@ +plan "unsupported-current" { + owner "receipt-worker" + current "0000" + version "0000" resource="file:0000.md" +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv index 583015f6..9f89efbc 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv @@ -1,18 +1,23 @@ path sha256 -experiment.tsv ea20f72c7221f6f2d7b41f4f2bdd1868960c0142584d76d553d172e4501fbad2 +experiment.tsv 0777e388a0bd37079084e3a550af9aa343f3f60c591266ff2a9ea0964d291232 arms.tsv 815eb439c6e4da9239e788a2de11794f81ec39d14f9e3061dc2973edd06b152f scenario.tsv b96c25507c1866427f29e4e594d24f78455b3b5a2ab3fefa785515cc7d580677 measurement-schema.tsv 7ce80573e9bf6957a0222e0417fc64e6592c29be9a981186327a33f6de1cbb48 receipt-schema.tsv 954c290299434e256fd03a0eef54938b3e2940b7919e1b5604986f0baa40c904 -blockers.tsv ff046fd9eff27f5fd9ef702e8d62a0693906564f67087608746c8099516d6c50 +blockers.tsv a8df999bdd1b588b24801c538246a4155e9bccae34f67ca822e9568094233c46 +runner.tsv 6e11100201af65c1b27e3b3540400497bb827f729b801aa870b21f1cd097bdbd arm-a/catalog/plans/receipt-report/lineage.tsv fe5ef897d0f4a00172ae08ee9024af45db6f7aa6915abe9aa66f78e2620ea162 +arm-a/catalog/plans/receipt-report/plan.kdl 1d5f559823f228896cce5e018c0745eca458267af636c94044de6a27f8f62029 arm-a/catalog/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-a/catalog/plans/receipt-report/versions/0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 +arm-a/catalog/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e arm-b/inbox/brief-0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-b/inbox/brief-0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 arm-b/inbox/thread.tsv 07aaee1152a5dd647cdd8cfb17e827cdc886d47936987c223a310a7f3375a86b arm-a/catalog/plans/receipt-report/receipts/README.md be10857144f3ef9185838396e799ba1a085c7d012c01f1f61ceee5bc489b2715 arm-b/receipts/README.md 4ad0cd92937a4a33f0202f3aaa94073d4d11a961dff3851007b4665b8e77900a +invalid/current/plan.kdl f6ac18a143e0a6ffb060a003208c64fd59dd52e7a90f5613de23b1aa82ce7a34 +invalid/current/0000.md 1a18971d45c564ee22de2496c1a086b6a799b9cf809a6d7351236d3c5b4e788a task-repo/package.json 8f756153985ab193ec21992e4adf874db4b02330db019006c5214550c2ff9618 task-repo/README.md c50a603eac8bb3dad994dfc96f682f7f8d9a56300e21a7f7be0d040308ff580b task-repo/src/report.mjs e2bc26ea2e19e14290585388c4f7d2cc10f56294cb4547068af884847197f020 @@ -21,4 +26,5 @@ reference/report.mjs 7343d91050754693f1ced37a97dae6f8701549af27ee78e08bab260a5d6 judges/report.test.mjs 24f41a2e596c940a8a3c5123e53b635bc66deb11d48bc0911a280d7a5485e670 judges/judge-task.sh 23680852e5d3ffd744eb00b6edf12c85f5560cbe3b406da45169e710eff2a10b run-recovery-matrix.sh 128e3caa9d125614bec70dc2ad3b7e5d1ef9f0ac442ea4dd54b9ad2f8245e5e6 +check-plan-contract.sh a1b225718ac2f9c9c4e7c5522398273d9cee081255fa39bcedbcdaeef36d8c17 task-repo.tree 22c368e3d2b9722c0fe53710dbd16e2010e98d7215db0aa42fe7532cfa9233c5 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv new file mode 100644 index 00000000..0ed7de7a --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv @@ -0,0 +1,9 @@ +field value +st2_pr https://github.com/compoundingtech/st2/pull/115 +source_full 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff +source_short 2caa0d7 +binary_sha256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 +source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 +runtime_scope read-only-validate-list-show-inspect +hosted_run https://github.com/compoundingtech/st2/actions/runs/30589548410 +hosted_status pass From 65b71af5ca48e84638fbae3edd2a60797147ee6e Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 31 Jul 2026 22:28:10 +0200 Subject: [PATCH 3/6] Report model-free catalog plan A/B result --- AGENT-SPEC.md | 12 + CATALOG.md | 2 +- cells/catalog-plan-vs-direct-brief/README.md | 36 ++- .../catalog-plan-vs-direct-brief.kdl | 30 +- .../agents/eval/receipt-worker/agent.kdl | 5 + .../plans/receipt-report/plan.kdl | 4 + .../plans/receipt-report/versions/0000.md | 37 +++ .../catalog/plans/receipt-report/lineage.tsv | 3 - .../plans/receipt-report/receipts/README.md | 6 - .../fixture/arm-b/inbox/thread.tsv | 3 - .../fixture/arm-b/receipts/README.md | 6 - .../fixture/arms.tsv | 6 +- .../fixture/blockers.tsv | 2 +- .../fixture/check-fixtures.sh | 89 ++++-- .../fixture/experiment.tsv | 3 +- .../fixture/measurement-schema.tsv | 26 +- .../fixture/provenance.tsv | 21 +- .../fixture/receipt-schema.tsv | 41 ++- .../fixture/run-recovery-matrix.sh | 299 +++++++++++++----- .../fixture/scenario.tsv | 12 +- 20 files changed, 440 insertions(+), 203 deletions(-) create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/versions/0000.md delete mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv delete mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md delete mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv delete mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 6c66b7fb..4afa722f 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -247,6 +247,13 @@ Prefer a catalog-owned `.st2/` overlay and locally excluded tool loaders. ## Experimental read-only plans +This candidate contract is not part of the released `0fed14b` corpus +conformance claim above. The paired model-free experiment is isolated to +[`catalog-plan-vs-direct-brief`](cells/catalog-plan-vs-direct-brief/) and runs +against st2 draft PR +[#115](https://github.com/compoundingtech/st2/pull/115) at exact source +`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`. + The experimental plan surface is pinned to the [source sketch revision `5c1d142`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994). An external plan is a top-level declaration: @@ -302,6 +309,11 @@ Global `--catalog` selects the same input. The experiment has no current pointer, execution, controller, schedule, step graph, retry, claim, receipt, event, reconciliation mutation, agent restart, CAS, merge, or deployment behavior. Direct KDL and direct human-to-agent planning remain supported. +The current model-free A/B finds no plan advantage for cold resume, exact +intent recovery, or acceptance evidence: both durable paths recover equally, +and neither surface reports worker acceptance. Static validation and resolved +provenance are plan-only authoring evidence; live correctness, traffic, cost, +and duration remain unresolved. ## Validation, health, and lifecycle diff --git a/CATALOG.md b/CATALOG.md index 9bd0a361..6b2534eb 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -27,7 +27,7 @@ harness-native loader plus canonical hook file. | `assignment-contract-hot-assignment` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `assignment-contract-hot-focus` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `assignment-contract-hot-resources` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | -| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 15 | — | **NO STRUCTURED RUN** | +| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 21 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | diff --git a/cells/catalog-plan-vs-direct-brief/README.md b/cells/catalog-plan-vs-direct-brief/README.md index cc8ba492..1124ad13 100644 --- a/cells/catalog-plan-vs-direct-brief/README.md +++ b/cells/catalog-plan-vs-direct-brief/README.md @@ -11,17 +11,28 @@ revision. Both arms receive durable local input and use the same evaluator-owned receipt fields, so direct planning remains a valid control rather than an intentionally fragile baseline. -The deterministic fixture proves: +The deterministic fixture now runs both durable product paths: - the seeded repository needs a real implementation and the arm-neutral reference solution passes public and held-out correctness tests; -- both arms recover the exact initial intent after a cold restart while their - remote source is unavailable; -- both arms receive and recover the exact steered intent; -- correctness, restart, partition, and steering are gating outcomes, while - coordination traffic and model cost remain reported comparison metrics; and +- arm A recovers an initial and then revised catalog snapshot through exact + `st2 plan show`/`inspect` calls; +- arm B receives the byte-identical revisions through real isolated + `st2 message send` deliveries with `inReplyTo` lineage, then recovers them + through `message ls`/`read`; +- both arms recover the same intent after cold state loss with their remote + source offline, so cold resume, intent recovery, and steering are ties; +- neither read-only plan inspection nor direct-message delivery reports worker + acceptance, so acceptance evidence is evaluator-owned in both arms and is + also a tie; +- the plan adds native static validation and resolved provenance, but the + current model-free result is `no-measured-advantage`; and - no model or provider is launched. +Correctness, coordination traffic, token use, cost, and wall duration remain +unresolved live-run endpoints. The reference solution establishes a valid task +and neutral judge; it is not substituted for two agent executions. + Arm A uses the exact external `plan.kdl` and agent `plan-ref` contract from [st2 draft PR #115](https://github.com/compoundingtech/st2/pull/115) at source `2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`, paired to the source gist at @@ -31,8 +42,11 @@ artifact SHA256 is The fixture exercises only `plan validate`, `list`, `show`, and `inspect`, and proves they do not alter the catalog. -Arm B retains equivalent durable message-thread metadata and receipt space. -The product experiment adds no current pointer, execution, scheduling, steps, -retries, progress claims, events, reconciliation, or CAS. A paid/live A/B -remains a separate authorization with an exact model, effort, budget, run -order, and rollback. +Arm B uses the product's ordinary durable message store rather than a synthetic +thread sidecar. Neither arm receives a pre-authored acceptance receipt. The +experiment uses plain copied folders, requires no server, and sets +`casRequired` to false. The product experiment adds no current pointer, +execution, scheduling, steps, retries, progress claims, events, +reconciliation, or CAS. A provider-backed A/B remains a separate authorization +with an exact model, effort, budget, run order, spend ceiling, cleanup, and +rollback. diff --git a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl index 9832e459..8bf879f2 100644 --- a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl +++ b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl @@ -17,14 +17,26 @@ eval { judge "PARITY — task intent, tools, budgets, judges, and done condition match across arms" { exec "grep -Fqx ARM-PARITY-GREEN-43af $RUNS_DIR/fixture-contract.out" } - judge "RESTART — both durable inputs recover the exact accepted revision after cold state loss" { - exec "grep -Fqx COLD-RESTART-GREEN-43af $RUNS_DIR/fixture-contract.out" + judge "RESTART — catalog plan and real direct-message inbox both recover after cold state loss" { + exec "grep -Fqx COLD-RESTART-TIE-43af $RUNS_DIR/fixture-contract.out" } - judge "PARTITION — recovery uses each arm's local durable source with its remote source unavailable" { - exec "grep -Fqx LOCAL-PARTITION-GREEN-43af $RUNS_DIR/fixture-contract.out" + judge "PARTITION — both arms recover locally while their remote sources are unavailable" { + exec "grep -Fqx LOCAL-PARTITION-TIE-43af $RUNS_DIR/fixture-contract.out" } - judge "STEERING — a complete new intent version and a direct follow-up carry identical revised intent" { - exec "grep -Fqx HUMAN-STEERING-GREEN-43af $RUNS_DIR/fixture-contract.out" + judge "INTENT — both arms recover byte-identical initial and revised intent" { + exec "grep -Fqx INTENT-RECOVERY-TIE-43af $RUNS_DIR/fixture-contract.out" + } + judge "STEERING — a complete plan version and reply-linked direct brief recover equally" { + exec "grep -Fqx HUMAN-STEERING-TIE-43af $RUNS_DIR/fixture-contract.out" + } + judge "ACCEPTANCE — neither read-only plan inspection nor message delivery proves worker acceptance" { + exec "grep -Fqx ACCEPTANCE-EVIDENCE-TIE-43af $RUNS_DIR/fixture-contract.out" + } + judge "STATIC VALUE — plan validation and provenance are plan-only authoring evidence" { + exec "grep -Fqx PLAN-STATIC-EVIDENCE-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "DIRECT REALISM — the control uses durable st2 messages and reply lineage" { + exec "grep -Fqx DIRECT-MESSAGE-EVIDENCE-GREEN-43af $RUNS_DIR/fixture-contract.out" } judge "MEASUREMENT — correctness gates are separate from neutral traffic and cost observations" { exec "grep -Fqx MEASUREMENT-CONTRACT-GREEN-43af $RUNS_DIR/fixture-contract.out" @@ -35,6 +47,12 @@ eval { judge "PAIRING — exact experimental source and artifact are frozen while live execution stays blocked" { exec "grep -Fqx PRODUCT-PAIRING-GREEN-43af $RUNS_DIR/fixture-contract.out" } + judge "STORAGE — plain local folders suffice and CAS is not required" { + exec "grep -Fqx PLAIN-FOLDER-NO-CAS-GREEN-43af $RUNS_DIR/fixture-contract.out" + } + judge "VERDICT — the model-free endpoints show no plan advantage and live endpoints stay unresolved" { + exec "grep -Fqx MODEL-FREE-OUTCOME-GREEN-43af $RUNS_DIR/fixture-contract.out" + } judge "RUNNER — the exact experimental st2 binary matches the accepted source and SHA256" { exec "grep -Fqx PLAN-RUNNER-PIN-GREEN-43af $RUNS_DIR/plan-contract.out" } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl new file mode 100644 index 00000000..dacd0fe6 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl @@ -0,0 +1,5 @@ +agent "receipt-worker" { + host "eval" + command "true" + plan-ref "file:../../../plans/receipt-report/plan.kdl" +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl new file mode 100644 index 00000000..d1d055e4 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl @@ -0,0 +1,4 @@ +plan "receipt-report" { + owner "receipt-worker" + version "0000" resource="file:versions/0000.md" +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/versions/0000.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/versions/0000.md new file mode 100644 index 00000000..7f75d24e --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/versions/0000.md @@ -0,0 +1,37 @@ +# Receipt report: latest accepted pass and last run + +Revision: `0000` + +## Goal + +Implement the report core in `src/report.mjs`. The report must retain the most +recent accepted `PASS` for each cell independently from that cell's last run, +which may be a failure. + +## Scope and owner + +You own the frozen `receipt-report` repository for this task. Change only +`src/report.mjs`. + +## Required behavior + +- `parseJsonLines(text)` ignores blank lines and returns records in input order. +- Malformed JSON, non-object JSON, or a record missing non-empty `run_id`, + `cell`, or `result` fields throws a `TypeError` that identifies the input line. +- `summarize(records)` returns one row per cell, sorted by cell name. +- Each row is `{ cell, accepted_pass, last_run }`. +- `accepted_pass` is the complete most recent `PASS` record for the cell, or + `null` when the cell has no pass. +- `last_run` is the complete last record for the cell. +- Neither exported function mutates caller-owned records. + +## Invariants and allowed actions + +Use only the repository's existing Node, Bash, and Git tools. Do not access the +network or add dependencies. Do not weaken or replace tests. No action outside +the frozen repository is authorized. + +## Evidence and done condition + +Run `npm test`. Completion additionally requires the evaluator's held-out tests +to pass and a diff containing only `src/report.mjs`. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv deleted file mode 100644 index ed50f7c3..00000000 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/lineage.tsv +++ /dev/null @@ -1,3 +0,0 @@ -version parent reason body_sha256 -0000 - initial frozen repository task 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 -0001 0000 human steering tightens input validation without changing scope aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md deleted file mode 100644 index 6f521b6b..00000000 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/receipts/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Evaluator receipt destination - -Future A-arm runs write evaluator-owned receipts here using -`receipt-schema.tsv`. These files report observed experiment outcomes; they do -not define plan intent, grant authority, or imply that this model-free scaffold -ran an agent. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv deleted file mode 100644 index b81c6230..00000000 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/inbox/thread.tsv +++ /dev/null @@ -1,3 +0,0 @@ -revision parent reason body_sha256 -0000 - initial frozen repository task 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 -0001 0000 human steering tightens input validation without changing scope aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md deleted file mode 100644 index c12bbbca..00000000 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-b/receipts/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Evaluator receipt destination - -Future B-arm runs write evaluator-owned receipts here using -`receipt-schema.tsv`. These files report observed experiment outcomes; they do -not replace the durable direct brief, grant authority, or imply that this -model-free scaffold ran an agent. diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv index 13c0721c..a7111672 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv @@ -1,3 +1,3 @@ -arm input_kind initial_source local_resume_source steering_source receipt_surface allowed_tools wall_budget_seconds model_token_budget judge_profile done_condition -A versioned-catalog-plan arm-a/catalog/plans/receipt-report/versions/0000.md local catalog plan version plus evaluator receipt arm-a/catalog/plans/receipt-report/versions/0001.md arm-a/catalog/plans/receipt-report/receipts node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change -B durable-direct-brief arm-b/inbox/brief-0000.md local durable brief thread plus evaluator receipt arm-b/inbox/brief-0001.md arm-b/receipts node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change +arm input_kind initial_source local_resume_source steering_source acceptance_surface allowed_tools wall_budget_seconds model_token_budget judge_profile done_condition +A versioned-catalog-plan arm-a/catalog-initial/plans/receipt-report/versions/0000.md local plan catalog inspected through st2 plan arm-a/catalog/plans/receipt-report/versions/0001.md evaluator-owned receipt only node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change +B durable-direct-brief arm-b/inbox/brief-0000.md local st2 message inbox read through st2 message arm-b/inbox/brief-0001.md evaluator-owned receipt only node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change diff --git a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv index 2aa5e93e..b2023fb8 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv @@ -1,3 +1,3 @@ blocker_id owner dependency required_evidence status st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact source 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff and binary SHA256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 with focused model 3/3, CLI 2/2, full workspace tests and doctests, scoped Clippy, rustfmt, diff receipts, and hosted run 30589548410 green ready -live-ab evals-owner future paid/live execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked +live-ab evals-owner future provider-backed execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh index 5ee3a452..2f0f37c0 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh @@ -9,10 +9,11 @@ measurements="$root/measurement-schema.tsv" receipts="$root/receipt-schema.tsv" blockers="$root/blockers.tsv" runner="$root/runner.tsv" -lineage="$root/arm-a/catalog/plans/receipt-report/lineage.tsv" -thread="$root/arm-b/inbox/thread.tsv" plan_declaration="$root/arm-a/catalog/plans/receipt-report/plan.kdl" agent_declaration="$root/arm-a/catalog/agents/eval/receipt-worker/agent.kdl" +initial_plan_declaration="$root/arm-a/catalog-initial/plans/receipt-report/plan.kdl" +initial_agent_declaration="$root/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl" +initial_plan_0000="$root/arm-a/catalog-initial/plans/receipt-report/versions/0000.md" plan_0000="$root/arm-a/catalog/plans/receipt-report/versions/0000.md" plan_0001="$root/arm-a/catalog/plans/receipt-report/versions/0001.md" brief_0000="$root/arm-b/inbox/brief-0000.md" @@ -36,19 +37,17 @@ tree_hash() { } expect_header "$experiment" $'field\tvalue' -expect_header "$arms" $'arm\tinput_kind\tinitial_source\tlocal_resume_source\tsteering_source\treceipt_surface\tallowed_tools\twall_budget_seconds\tmodel_token_budget\tjudge_profile\tdone_condition' +expect_header "$arms" $'arm\tinput_kind\tinitial_source\tlocal_resume_source\tsteering_source\tacceptance_surface\tallowed_tools\twall_budget_seconds\tmodel_token_budget\tjudge_profile\tdone_condition' expect_header "$scenario" $'ordinal\tevent\tarm_a_input\tarm_b_input\tinvariant' -expect_header "$measurements" $'metric\ttype\tunit\trole\tpreferred_direction\tmodel_free_value\tlive_source' +expect_header "$measurements" $'metric\ttype\tunit\trole\tpreferred_direction\tmodel_free_arm_a\tmodel_free_arm_b\tmodel_free_verdict\tlive_source' expect_header "$receipts" $'field\ttype\trequired\tdescription' expect_header "$blockers" $'blocker_id\towner\tdependency\trequired_evidence\tstatus' expect_header "$runner" $'field\tvalue' -expect_header "$lineage" $'version\tparent\treason\tbody_sha256' -expect_header "$thread" $'revision\tparent\treason\tbody_sha256' test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$arms")" -eq 2 test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$scenario")" -eq 6 -test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$measurements")" -eq 10 -test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$receipts")" -eq 17 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$measurements")" -eq 14 +test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$receipts")" -eq 24 test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$blockers")" -eq 2 test "$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$runner")" -eq 8 @@ -58,11 +57,12 @@ grep -Fqx $'wall_budget_seconds\t1200' "$experiment" grep -Fqx $'model_token_budget\t40000' "$experiment" grep -Fqx $'initial_revision\t0000' "$experiment" grep -Fqx $'steering_revision\t0001' "$experiment" +grep -Fqx $'model_free_result\tno-measured-advantage-for-cold-resume-intent-recovery-or-acceptance-evidence' "$experiment" grep -Fqx $'source_gist\thttps://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" grep -Fqx $'st2_plan_source\t2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$experiment" grep -Fqx $'st2_plan_binary_sha256\t83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$experiment" -grep -Fqx $'live_run_status\tblocked-pending-separate-paid-authorization' "$experiment" +grep -Fqx $'live_run_status\tblocked-pending-separate-provider-authorization' "$experiment" grep -Fqx $'st2_pr\thttps://github.com/compoundingtech/st2/pull/115' "$runner" grep -Fqx $'source_full\t2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$runner" grep -Fqx $'source_short\t2caa0d7' "$runner" @@ -72,6 +72,7 @@ grep -Fqx $'runtime_scope\tread-only-validate-list-show-inspect' "$runner" grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30589548410' "$runner" grep -Fqx $'hosted_status\tpass' "$runner" +cmp -s "$initial_plan_0000" "$plan_0000" cmp -s "$plan_0000" "$brief_0000" cmp -s "$plan_0001" "$brief_0001" test "$(sha256sum "$plan_0000" | awk '{print $1}')" != \ @@ -92,23 +93,25 @@ awk -F '\t' ' } } ' "$arms" -grep -Fqx $'A\tversioned-catalog-plan\tarm-a/catalog/plans/receipt-report/versions/0000.md\tlocal catalog plan version plus evaluator receipt\tarm-a/catalog/plans/receipt-report/versions/0001.md\tarm-a/catalog/plans/receipt-report/receipts\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" -grep -Fqx $'B\tdurable-direct-brief\tarm-b/inbox/brief-0000.md\tlocal durable brief thread plus evaluator receipt\tarm-b/inbox/brief-0001.md\tarm-b/receipts\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" - -expected_0000="$(sha256sum "$plan_0000" | awk '{print $1}')" -expected_0001="$(sha256sum "$plan_0001" | awk '{print $1}')" -grep -Fqx "0000 - initial frozen repository task $expected_0000" "$lineage" -grep -Fqx "0001 0000 human steering tightens input validation without changing scope $expected_0001" "$lineage" -grep -Fqx "0000 - initial frozen repository task $expected_0000" "$thread" -grep -Fqx "0001 0000 human steering tightens input validation without changing scope $expected_0001" "$thread" +grep -Fqx $'A\tversioned-catalog-plan\tarm-a/catalog-initial/plans/receipt-report/versions/0000.md\tlocal plan catalog inspected through st2 plan\tarm-a/catalog/plans/receipt-report/versions/0001.md\tevaluator-owned receipt only\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" +grep -Fqx $'B\tdurable-direct-brief\tarm-b/inbox/brief-0000.md\tlocal st2 message inbox read through st2 message\tarm-b/inbox/brief-0001.md\tevaluator-owned receipt only\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" + grep -Fqx 'plan "receipt-report" {' "$plan_declaration" grep -Fqx ' owner "receipt-worker"' "$plan_declaration" grep -Fqx ' version "0000" resource="file:versions/0000.md"' "$plan_declaration" grep -Fqx ' version "0001" resource="file:versions/0001.md" {' "$plan_declaration" grep -Fqx ' parent "0000"' "$plan_declaration" grep -Fqx ' why "Human steering tightens input validation without changing scope."' "$plan_declaration" +grep -Fqx 'plan "receipt-report" {' "$initial_plan_declaration" +grep -Fqx ' owner "receipt-worker"' "$initial_plan_declaration" +grep -Fqx ' version "0000" resource="file:versions/0000.md"' "$initial_plan_declaration" +if grep -Fq 'version "0001"' "$initial_plan_declaration"; then + echo "initial plan snapshot already contains the steering revision" >&2 + exit 1 +fi grep -Fqx 'agent "receipt-worker" {' "$agent_declaration" grep -Fqx ' plan-ref "file:../../../plans/receipt-report/plan.kdl"' "$agent_declaration" +cmp -s "$initial_agent_declaration" "$agent_declaration" if ( cd "$root/task-repo" @@ -136,17 +139,42 @@ fi echo "ARM-PARITY-GREEN-43af" matrix_output="$(CATALOG="$root" "$root/run-recovery-matrix.sh")" -grep -Fqx 'RECOVERY-GREEN-A-43af' <<<"$matrix_output" -grep -Fqx 'RECOVERY-GREEN-B-43af' <<<"$matrix_output" -echo "COLD-RESTART-GREEN-43af" -echo "LOCAL-PARTITION-GREEN-43af" -echo "HUMAN-STEERING-GREEN-43af" - -test "$(awk -F '\t' 'NR > 1 && $4 == "gating" { count++ } END { print count + 0 }' "$measurements")" -eq 4 -test "$(awk -F '\t' 'NR > 1 && $4 == "reporting" { count++ } END { print count + 0 }' "$measurements")" -eq 6 +for marker in \ + COLD-RESTART-TIE-43af \ + LOCAL-PARTITION-TIE-43af \ + INTENT-RECOVERY-TIE-43af \ + HUMAN-STEERING-TIE-43af \ + ACCEPTANCE-EVIDENCE-TIE-43af \ + PLAN-STATIC-EVIDENCE-GREEN-43af \ + DIRECT-MESSAGE-EVIDENCE-GREEN-43af \ + MODEL-FREE-COMPARISON-GREEN-43af \ + PLAIN-FOLDER-NO-CAS-GREEN-43af; do + grep -Fqx "$marker" <<<"$matrix_output" +done +jq -e ' + .outcomes.coldResume.verdict == "tie" and + .outcomes.intentRecovery.verdict == "tie" and + .outcomes.humanSteering.verdict == "tie" and + .outcomes.acceptanceEvidence.verdict == "tie" and + .conclusion == "no-measured-advantage" +' "$root/comparison-receipt.json" >/dev/null +echo "COLD-RESTART-TIE-43af" +echo "LOCAL-PARTITION-TIE-43af" +echo "INTENT-RECOVERY-TIE-43af" +echo "HUMAN-STEERING-TIE-43af" +echo "ACCEPTANCE-EVIDENCE-TIE-43af" +echo "PLAN-STATIC-EVIDENCE-GREEN-43af" +echo "DIRECT-MESSAGE-EVIDENCE-GREEN-43af" +echo "PLAIN-FOLDER-NO-CAS-GREEN-43af" +echo "MODEL-FREE-OUTCOME-GREEN-43af" + +test "$(awk -F '\t' 'NR > 1 && $4 == "model-free-gating" { count++ } END { print count + 0 }' "$measurements")" -eq 3 +test "$(awk -F '\t' 'NR > 1 && $4 == "live-gating" { count++ } END { print count + 0 }' "$measurements")" -eq 2 +test "$(awk -F '\t' 'NR > 1 && $4 == "reporting" { count++ } END { print count + 0 }' "$measurements")" -eq 3 +test "$(awk -F '\t' 'NR > 1 && $4 == "live-reporting" { count++ } END { print count + 0 }' "$measurements")" -eq 6 awk -F '\t' ' - NR > 1 && $4 == "gating" && $5 != "pass" { exit 1 } - NR > 1 && $4 == "reporting" && $5 != "lower" { exit 1 } + NR > 1 && ($4 == "model-free-gating" || $4 == "live-gating" || $4 == "reporting") && $5 != "pass" && $1 != "acceptance_evidence" { exit 1 } + NR > 1 && $4 == "live-reporting" && $5 != "lower" { exit 1 } ' "$measurements" test "$(awk -F '\t' 'NR > 1 && $3 != "yes" { count++ } END { print count + 0 }' "$receipts")" -eq 0 echo "MEASUREMENT-CONTRACT-GREEN-43af" @@ -167,7 +195,8 @@ while IFS=$'\t' read -r path expected; do test "$actual" = "$expected" checked=$((checked + 1)) done <"$root/provenance.tsv" -test "$checked" -eq 29 +expected_provenance="$(awk -F '\t' 'NR > 1 { count++ } END { print count + 0 }' "$root/provenance.tsv")" +test "$checked" -eq "$expected_provenance" echo "PROVENANCE-GREEN-43af" test "$(awk -F '\t' 'NR > 1 && $5 == "ready" { count++ } END { print count + 0 }' "$blockers")" -eq 1 @@ -176,7 +205,7 @@ grep -Fq '2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$blockers" grep -Fq '83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$blockers" grep -Fq 'hosted run 30589548410 green' "$blockers" grep -Fq 'separate authorization' "$blockers" -test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 2 +test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 4 echo "PRODUCT-PAIRING-GREEN-43af" if rg -n --pcre2 \ diff --git a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv index 906ccddc..e5fdc5a4 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv @@ -16,4 +16,5 @@ judge_profile arm-neutral-public-plus-held-out done_condition public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change initial_revision 0000 steering_revision 0001 -live_run_status blocked-pending-separate-paid-authorization +model_free_result no-measured-advantage-for-cold-resume-intent-recovery-or-acceptance-evidence +live_run_status blocked-pending-separate-provider-authorization diff --git a/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv b/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv index ff49ab3b..eb336298 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/measurement-schema.tsv @@ -1,11 +1,15 @@ -metric type unit role preferred_direction model_free_value live_source -correctness boolean pass/fail gating pass reference-pass arm-neutral held-out judge -cold_restart_recovery boolean pass/fail gating pass pass evaluator recovery receipt -partition_recovery boolean pass/fail gating pass pass evaluator recovery receipt -human_steering boolean pass/fail gating pass pass evaluator steering receipt -coordination_messages integer count reporting lower unmeasured st2 message receipt -coordination_bytes integer bytes reporting lower unmeasured st2 message and sync receipt -model_input_tokens integer tokens reporting lower 0 harness usage receipt -model_output_tokens integer tokens reporting lower 0 harness usage receipt -cost_usd decimal USD reporting lower 0 harness usage receipt -wall_duration decimal seconds reporting lower unmeasured evaluator run receipt +metric type unit role preferred_direction model_free_arm_a model_free_arm_b model_free_verdict live_source +correctness boolean pass/fail live-gating pass not-run not-run unresolved-live arm-neutral held-out judge +cold_restart_recovery boolean pass/fail model-free-gating pass pass pass tie comparison receipt +partition_recovery boolean pass/fail model-free-gating pass pass pass tie comparison receipt +human_steering boolean pass/fail model-free-gating pass pass pass tie comparison receipt +acceptance_evidence enum evaluator-only|product-native live-gating product-native evaluator-only evaluator-only tie comparison receipt plus future worker claim +static_intent_validation boolean pass/fail reporting pass pass n/a plan-only st2 plan validate +lineage_observation boolean pass/fail reporting pass pass pass tie st2 plan show and st2 message inReplyTo +provenance_resolution boolean pass/fail reporting pass pass n/a plan-only st2 plan inspect +coordination_messages integer count live-reporting lower unmeasured unmeasured unresolved-live st2 message receipt +coordination_bytes integer bytes live-reporting lower unmeasured unmeasured unresolved-live st2 message and sync receipt +model_input_tokens integer tokens live-reporting lower 0 0 model-free-only harness usage receipt +model_output_tokens integer tokens live-reporting lower 0 0 model-free-only harness usage receipt +cost_usd decimal USD live-reporting lower 0 0 model-free-only harness usage receipt +wall_duration decimal seconds live-reporting lower unmeasured unmeasured unresolved-live evaluator run receipt diff --git a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv index 9f89efbc..5cd290bf 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv @@ -1,21 +1,20 @@ path sha256 -experiment.tsv 0777e388a0bd37079084e3a550af9aa343f3f60c591266ff2a9ea0964d291232 -arms.tsv 815eb439c6e4da9239e788a2de11794f81ec39d14f9e3061dc2973edd06b152f -scenario.tsv b96c25507c1866427f29e4e594d24f78455b3b5a2ab3fefa785515cc7d580677 -measurement-schema.tsv 7ce80573e9bf6957a0222e0417fc64e6592c29be9a981186327a33f6de1cbb48 -receipt-schema.tsv 954c290299434e256fd03a0eef54938b3e2940b7919e1b5604986f0baa40c904 -blockers.tsv a8df999bdd1b588b24801c538246a4155e9bccae34f67ca822e9568094233c46 +experiment.tsv d0cd5fe98e274c3e3ba5492d25553669e4f520c0170f61ee6fcdadd8a98ae48a +arms.tsv 29572c02d123f4097aaf69b6fb8153d60190adecbcdcd75fdfad3690ca99df2f +scenario.tsv 884a9c83b983128c8e642bf3a945d6f7edb544ac224f5934a4468f17a48d282b +measurement-schema.tsv 01763a0c090b9c2fd39e5f1b7995380e2dc0d2e5d4230227749ddd41a133f9ab +receipt-schema.tsv c996eaee81301640442b04d9fca31937bb840ed10ef78cca035e5990d1fe6441 +blockers.tsv 00b95bcb9bb3b308fda5c529072686e658dbcf9a6965eefd852679e823e30cc0 runner.tsv 6e11100201af65c1b27e3b3540400497bb827f729b801aa870b21f1cd097bdbd -arm-a/catalog/plans/receipt-report/lineage.tsv fe5ef897d0f4a00172ae08ee9024af45db6f7aa6915abe9aa66f78e2620ea162 +arm-a/catalog-initial/plans/receipt-report/plan.kdl 3c883689f3709c7592c226c513af944a1036682612bfc7b0576dd1761de81a4a +arm-a/catalog-initial/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e arm-a/catalog/plans/receipt-report/plan.kdl 1d5f559823f228896cce5e018c0745eca458267af636c94044de6a27f8f62029 arm-a/catalog/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-a/catalog/plans/receipt-report/versions/0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 arm-a/catalog/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e arm-b/inbox/brief-0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-b/inbox/brief-0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 -arm-b/inbox/thread.tsv 07aaee1152a5dd647cdd8cfb17e827cdc886d47936987c223a310a7f3375a86b -arm-a/catalog/plans/receipt-report/receipts/README.md be10857144f3ef9185838396e799ba1a085c7d012c01f1f61ceee5bc489b2715 -arm-b/receipts/README.md 4ad0cd92937a4a33f0202f3aaa94073d4d11a961dff3851007b4665b8e77900a invalid/current/plan.kdl f6ac18a143e0a6ffb060a003208c64fd59dd52e7a90f5613de23b1aa82ce7a34 invalid/current/0000.md 1a18971d45c564ee22de2496c1a086b6a799b9cf809a6d7351236d3c5b4e788a task-repo/package.json 8f756153985ab193ec21992e4adf874db4b02330db019006c5214550c2ff9618 @@ -25,6 +24,6 @@ task-repo/test/report.test.mjs e9008004b55687df44025f66abe2e5ff739de8d8c7bec27d8 reference/report.mjs 7343d91050754693f1ced37a97dae6f8701549af27ee78e08bab260a5d6550c9 judges/report.test.mjs 24f41a2e596c940a8a3c5123e53b635bc66deb11d48bc0911a280d7a5485e670 judges/judge-task.sh 23680852e5d3ffd744eb00b6edf12c85f5560cbe3b406da45169e710eff2a10b -run-recovery-matrix.sh 128e3caa9d125614bec70dc2ad3b7e5d1ef9f0ac442ea4dd54b9ad2f8245e5e6 +run-recovery-matrix.sh 8d1ddf23ecde6ba0d0ed763b71f690b962c66e0e80f72d35b0580e27dc5c1c87 check-plan-contract.sh a1b225718ac2f9c9c4e7c5522398273d9cee081255fa39bcedbcdaeef36d8c17 task-repo.tree 22c368e3d2b9722c0fe53710dbd16e2010e98d7215db0aa42fe7532cfa9233c5 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv b/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv index 4b1b3c78..63d79839 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/receipt-schema.tsv @@ -1,18 +1,25 @@ field type required description -receipt_id string yes evaluator-generated unique run receipt id -arm enum:A|B yes experiment arm -phase enum:accepted|recovered|steered|completed yes observed evaluator phase -intent_revision string yes exact accepted intent revision -intent_sha256 sha256 yes exact intent body hash -seed_repo_sha256 sha256 yes frozen repository tree hash -correctness enum:pass|fail|not-run yes arm-neutral judge outcome -cold_restart_recovery enum:pass|fail|not-run yes exact local intent recovery outcome -partition_recovery enum:pass|fail|not-run yes recovery while remote input is unavailable -human_steering enum:pass|fail|not-run yes revised intent acceptance outcome -coordination_messages nonnegative-integer yes observed external message count -coordination_bytes nonnegative-integer yes observed message and sync payload bytes -model_input_tokens nonnegative-integer yes harness-reported input tokens -model_output_tokens nonnegative-integer yes harness-reported output tokens -cost_usd nonnegative-decimal yes harness-reported cost -wall_duration_seconds nonnegative-decimal yes evaluator elapsed wall time -evidence string yes path or immutable URL for supporting output +schemaVersion integer yes comparison receipt schema version +experimentId string yes frozen experiment identity +runner.source git-sha yes exact experimental st2 source +runner.sha256 sha256 yes exact experimental st2 executable +modelCalls nonnegative-integer yes provider/model calls made by this run +remoteSourcesOfflineDuringRecovery boolean yes both recoveries occurred with remote sources unavailable +storage.plainFoldersSufficient boolean yes no service or content-addressed store is required +storage.casRequired boolean yes whether content-addressed storage is required +arms.A.transport enum:catalog-plan yes arm A durable transport +arms.B.transport enum:st2-direct-messages yes arm B durable transport +arms.A.initialIntentSha256 sha256 yes recovered initial intent hash +arms.B.initialIntentSha256 sha256 yes recovered initial intent hash +arms.A.steeredIntentSha256 sha256 yes recovered revised intent hash +arms.B.steeredIntentSha256 sha256 yes recovered revised intent hash +outcomes.coldResume.verdict enum:tie|arm-a|arm-b yes comparative cold-resume result +outcomes.intentRecovery.verdict enum:tie|arm-a|arm-b yes comparative exact-intent result +outcomes.humanSteering.verdict enum:tie|arm-a|arm-b yes comparative steering result +outcomes.acceptanceEvidence.verdict enum:tie|arm-a|arm-b yes comparative worker-acceptance evidence result +planOnly.staticIntentValidation enum:pass|fail yes read-only plan schema and graph validation +planOnly.provenanceResolution enum:pass|fail yes read-only resolved resource provenance +directOnly.messageDelivery enum:pass|fail yes durable direct-message delivery +directOnly.replyLineage enum:pass|fail yes direct-message inReplyTo lineage +liveUnresolved array yes endpoints requiring separately authorized model runs +conclusion enum:no-measured-advantage|arm-a-advantage|arm-b-advantage yes model-free comparative conclusion diff --git a/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh b/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh index 20ecd1f4..ef9398c5 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh @@ -2,100 +2,225 @@ set -euo pipefail root="${CATALOG:?CATALOG must be set}" +runner="$root/runner.tsv" +receipt="$root/comparison-receipt.json" +st2_path="$(command -v st2)" scratch="$(mktemp -d)" + cleanup() { rm -rf -- "$scratch" } trap cleanup EXIT -tree_hash() { - ( - cd "$1" - find . -type f -print0 | - LC_ALL=C sort -z | - xargs -0 sha256sum | - sha256sum | - awk '{print $1}' - ) +runner_value() { + awk -F '\t' -v key="$1" '$1 == key { print $2 }' "$runner" +} + +hash_file() { + sha256sum "$1" | awk '{print $1}' } -seed_hash="$(tree_hash "$root/task-repo")" -receipt_header=$'receipt_id\tarm\tphase\tintent_revision\tintent_sha256\tseed_repo_sha256\tcorrectness\tcold_restart_recovery\tpartition_recovery\thuman_steering\tcoordination_messages\tcoordination_bytes\tmodel_input_tokens\tmodel_output_tokens\tcost_usd\twall_duration_seconds\tevidence' - -run_arm() { - arm="$1" - remote="$scratch/$arm/remote" - local_store="$scratch/$arm/local" - session="$scratch/$arm/session" - receipts="$scratch/$arm/receipts.tsv" - - mkdir -p "$remote" "$local_store" "$session" - printf '%s\n' "$receipt_header" >"$receipts" - - case "$arm" in - A) - cp -a "$root/arm-a/catalog" "$remote/catalog" - initial_source="$remote/catalog/plans/receipt-report/versions/0000.md" - steered_source="$remote/catalog/plans/receipt-report/versions/0001.md" - initial_local="$local_store/catalog/plans/receipt-report/versions/0000.md" - steered_local="$local_store/catalog/plans/receipt-report/versions/0001.md" - mkdir -p "$(dirname "$initial_local")" - ;; - B) - cp -a "$root/arm-b/inbox" "$remote/inbox" - initial_source="$remote/inbox/brief-0000.md" - steered_source="$remote/inbox/brief-0001.md" - initial_local="$local_store/inbox/brief-0000.md" - steered_local="$local_store/inbox/brief-0001.md" - mkdir -p "$(dirname "$initial_local")" - ;; - *) - echo "unknown arm: $arm" >&2 - return 2 - ;; - esac - - initial_hash="$(sha256sum "$initial_source" | awk '{print $1}')" - steered_hash="$(sha256sum "$steered_source" | awk '{print $1}')" - cp "$initial_source" "$initial_local" - printf '%s\n' \ - "${arm}-accepted-0000 ${arm} accepted 0000 ${initial_hash} ${seed_hash} not-run not-run not-run not-run 0 0 0 0 0 0 local:${initial_local#$scratch/}" \ - >>"$receipts" - - printf 'volatile state\n' >"$session/state" - rm -rf -- "$session" - mv "$remote" "$remote.offline" - test ! -e "$remote" - test "$(sha256sum "$initial_local" | awk '{print $1}')" = "$initial_hash" - printf '%s\n' \ - "${arm}-recovered-0000 ${arm} recovered 0000 ${initial_hash} ${seed_hash} not-run pass pass not-run 0 0 0 0 0 0 local:${initial_local#$scratch/}" \ - >>"$receipts" - - mv "$remote.offline" "$remote" - cp "$steered_source" "$steered_local" - printf '%s\n' \ - "${arm}-steered-0001 ${arm} steered 0001 ${steered_hash} ${seed_hash} not-run pass not-run pass 0 0 0 0 0 0 local:${steered_local#$scratch/}" \ - >>"$receipts" - - mkdir -p "$session" - printf 'volatile state\n' >"$session/state" - rm -rf -- "$session" - mv "$remote" "$remote.offline" - test ! -e "$remote" - test "$(sha256sum "$steered_local" | awk '{print $1}')" = "$steered_hash" - test "$(sha256sum "$initial_local" | awk '{print $1}')" = "$initial_hash" - printf '%s\n' \ - "${arm}-recovered-0001 ${arm} recovered 0001 ${steered_hash} ${seed_hash} not-run pass pass pass 0 0 0 0 0 0 local:${steered_local#$scratch/}" \ - >>"$receipts" - - awk -F '\t' ' - NR == 1 { expected = 17 } - NF != expected { exit 1 } - NR > 1 && $2 != "'"$arm"'" { exit 1 } - END { if (NR != 5) exit 1 } - ' "$receipts" - echo "RECOVERY-GREEN-${arm}-43af" +assert_no_acceptance_claim() { + jq -e ' + (has("accepted") | not) and + (has("acceptance") | not) and + (has("completed") | not) and + (has("receipt") | not) and + (has("receipts") | not) + ' >/dev/null } -run_arm A -run_arm B +source_full="$(runner_value source_full)" +source_short="$(runner_value source_short)" +expected_binary_sha="$(runner_value binary_sha256)" +version_regex="^st2 0\\.1\\.0 — running from local source \\(${source_short}, .+ ago\\)$" +actual_version="$($st2_path --version)" +actual_binary_sha="$(hash_file "$st2_path")" +[[ "$actual_version" =~ $version_regex ]] +test "$actual_binary_sha" = "$expected_binary_sha" + +remote="$scratch/remote" +local_store="$scratch/local" +session="$scratch/session" +plan_local="$local_store/plan-catalog" +direct_bus="$local_store/direct-bus" +mkdir -p "$remote" "$local_store" "$session" +cp -a "$root/arm-a/catalog-initial" "$remote/plan-initial" +cp -a "$root/arm-a/catalog" "$remote/plan-steered" +cp -a "$root/arm-b/inbox" "$remote/direct-source" + +# Initial delivery: sync only the initial plan snapshot, and deliver only the +# initial direct brief through the real st2 message store. +cp -a "$remote/plan-initial" "$plan_local" +initial_message="$($st2_path message send eval.worker \ + --catalog "$direct_bus" \ + --as eval.supervisor \ + --subject 'intent revision 0000' \ + <"$remote/direct-source/brief-0000.md")" +test -n "$initial_message" + +# Cold restart and partition: volatile state disappears and both remote input +# sources go offline. Recovery must use only each arm's local durable store. +printf 'volatile\n' >"$session/state" +rm -rf -- "$session" +mv "$remote" "$remote.offline" +test ! -e "$remote" + +plan_initial_show="$($st2_path plan show receipt-report --catalog "$plan_local" --json)" +plan_initial_inspect="$($st2_path plan inspect receipt-report --catalog "$plan_local" --json)" +jq -e ' + .frontier == ["0000"] and + (.versions | length) == 1 and + .versions[0].identity == "0000" +' <<<"$plan_initial_show" >/dev/null +plan_initial_path="$(jq -r '.versions[0].resolvedResource' <<<"$plan_initial_inspect")" +plan_initial_sha="$(hash_file "$plan_initial_path")" +test "$plan_initial_sha" = "$(hash_file "$root/arm-a/catalog-initial/plans/receipt-report/versions/0000.md")" +assert_no_acceptance_claim <<<"$plan_initial_show" +assert_no_acceptance_claim <<<"$plan_initial_inspect" + +direct_initial_list="$($st2_path message ls eval.worker --catalog "$direct_bus" --json)" +jq -e --arg filename "$initial_message" ' + length == 1 and + .[0].filename == $filename and + .[0].from == "eval.supervisor" and + .[0].subject == "intent revision 0000" and + .[0].inReplyTo == null +' <<<"$direct_initial_list" >/dev/null +direct_initial_read="$($st2_path message read eval.worker "$initial_message" --catalog "$direct_bus" --json)" +jq -rj '.body' <<<"$direct_initial_read" >"$scratch/direct-initial.md" +direct_initial_sha="$(hash_file "$scratch/direct-initial.md")" +test "$direct_initial_sha" = "$(hash_file "$root/arm-b/inbox/brief-0000.md")" +assert_no_acceptance_claim <<<"$direct_initial_read" +echo "COLD-RESTART-TIE-43af" +echo "LOCAL-PARTITION-TIE-43af" + +# Human steering: restore the remote source, sync a complete new plan snapshot, +# and deliver a reply-linked direct brief. Then repeat cold recovery offline. +mv "$remote.offline" "$remote" +rm -rf -- "$plan_local" +cp -a "$remote/plan-steered" "$plan_local" +steered_message="$($st2_path message send eval.worker \ + --catalog "$direct_bus" \ + --as eval.supervisor \ + --subject 'intent revision 0001' \ + --in-reply-to "$initial_message" \ + <"$remote/direct-source/brief-0001.md")" +test -n "$steered_message" + +mkdir -p "$session" +printf 'volatile\n' >"$session/state" +rm -rf -- "$session" +mv "$remote" "$remote.offline" +test ! -e "$remote" + +plan_steered_show="$($st2_path plan show receipt-report --catalog "$plan_local" --json)" +plan_steered_inspect="$($st2_path plan inspect receipt-report --catalog "$plan_local" --json)" +jq -e ' + .frontier == ["0001"] and + (.versions | length) == 2 and + .versions[0].identity == "0000" and + .versions[1].identity == "0001" and + .versions[1].parents == ["0000"] +' <<<"$plan_steered_show" >/dev/null +plan_steered_path="$(jq -r '.versions[] | select(.identity == "0001") | .resolvedResource' <<<"$plan_steered_inspect")" +plan_steered_sha="$(hash_file "$plan_steered_path")" +test "$plan_steered_sha" = "$(hash_file "$root/arm-a/catalog/plans/receipt-report/versions/0001.md")" +assert_no_acceptance_claim <<<"$plan_steered_show" +assert_no_acceptance_claim <<<"$plan_steered_inspect" + +direct_steered_list="$($st2_path message ls eval.worker --catalog "$direct_bus" --json)" +jq -e --arg initial "$initial_message" --arg steered "$steered_message" ' + length == 2 and + any(.[]; .filename == $initial and .inReplyTo == null) and + any(.[]; .filename == $steered and .inReplyTo == $initial and .subject == "intent revision 0001") +' <<<"$direct_steered_list" >/dev/null +direct_steered_read="$($st2_path message read eval.worker "$steered_message" --catalog "$direct_bus" --json)" +jq -rj '.body' <<<"$direct_steered_read" >"$scratch/direct-steered.md" +direct_steered_sha="$(hash_file "$scratch/direct-steered.md")" +test "$direct_steered_sha" = "$(hash_file "$root/arm-b/inbox/brief-0001.md")" +test "$(jq -r '.inReplyTo' <<<"$direct_steered_read")" = "$initial_message" +assert_no_acceptance_claim <<<"$direct_steered_read" + +test "$plan_initial_sha" = "$direct_initial_sha" +test "$plan_steered_sha" = "$direct_steered_sha" +echo "INTENT-RECOVERY-TIE-43af" +echo "HUMAN-STEERING-TIE-43af" +echo "ACCEPTANCE-EVIDENCE-TIE-43af" +echo "PLAN-STATIC-EVIDENCE-GREEN-43af" +echo "DIRECT-MESSAGE-EVIDENCE-GREEN-43af" + +jq -n \ + --arg experiment_id "catalog-plan-vs-direct-brief-v1" \ + --arg source "$source_full" \ + --arg sha "$actual_binary_sha" \ + --arg initial_sha "$plan_initial_sha" \ + --arg steered_sha "$plan_steered_sha" \ + '{ + schemaVersion: 1, + experimentId: $experiment_id, + runner: {source: $source, sha256: $sha}, + modelCalls: 0, + remoteSourcesOfflineDuringRecovery: true, + storage: {plainFoldersSufficient: true, casRequired: false}, + arms: { + A: { + transport: "catalog-plan", + initialIntentSha256: $initial_sha, + steeredIntentSha256: $steered_sha + }, + B: { + transport: "st2-direct-messages", + initialIntentSha256: $initial_sha, + steeredIntentSha256: $steered_sha + } + }, + outcomes: { + coldResume: {A: "pass", B: "pass", verdict: "tie"}, + intentRecovery: {A: "pass", B: "pass", verdict: "tie"}, + humanSteering: {A: "pass", B: "pass", verdict: "tie"}, + acceptanceEvidence: { + A: "evaluator-only", + B: "evaluator-only", + verdict: "tie", + reason: "Neither read-only plan inspection nor direct-message delivery reports worker acceptance." + } + }, + planOnly: { + staticIntentValidation: "pass", + provenanceResolution: "pass" + }, + directOnly: { + messageDelivery: "pass", + replyLineage: "pass" + }, + liveUnresolved: [ + "correctness", + "coordination_messages", + "coordination_bytes", + "model_input_tokens", + "model_output_tokens", + "cost_usd", + "wall_duration" + ], + conclusion: "no-measured-advantage" + }' >"$receipt" + +jq -e ' + .modelCalls == 0 and + .remoteSourcesOfflineDuringRecovery == true and + .storage.plainFoldersSufficient == true and + .storage.casRequired == false and + .outcomes.coldResume.verdict == "tie" and + .outcomes.intentRecovery.verdict == "tie" and + .outcomes.humanSteering.verdict == "tie" and + .outcomes.acceptanceEvidence.verdict == "tie" and + .planOnly.staticIntentValidation == "pass" and + .planOnly.provenanceResolution == "pass" and + .directOnly.messageDelivery == "pass" and + .directOnly.replyLineage == "pass" and + .conclusion == "no-measured-advantage" +' "$receipt" >/dev/null + +echo "MODEL-FREE-COMPARISON-GREEN-43af" +echo "PLAIN-FOLDER-NO-CAS-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv b/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv index 021b7bf0..221af70e 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/scenario.tsv @@ -1,7 +1,7 @@ ordinal event arm_a_input arm_b_input invariant -1 initial_delivery versions/0000.md brief-0000.md byte-identical intent and one durable acceptance receipt -2 cold_restart local version 0000 local brief 0000 no volatile conversation or process state -3 simulated_partition local catalog only local durable inbox only remote input source unavailable during recovery -4 human_steering versions/0001.md brief-0001.md byte-identical complete revised intent -5 restart_after_steering local versions 0000 and 0001 local briefs 0000 and 0001 revision 0001 selected and revision 0000 retained -6 future_completion same arm-neutral judge same arm-neutral judge report correctness recovery traffic and cost without preferred-arm thresholds +1 initial_delivery catalog snapshot with version 0000 st2 message carrying brief 0000 byte-identical intent; neither delivery claims worker acceptance +2 cold_restart st2 plan show on local snapshot st2 message read from local inbox no volatile conversation or process state +3 simulated_partition local catalog only local message inbox only remote input source unavailable during recovery +4 human_steering catalog snapshot adds version 0001 second st2 message replies to brief 0000 byte-identical complete revised intent +5 restart_after_steering frontier 0001 with version 0000 retained brief 0001 with inReplyTo 0000 retained revision 0001 recovered and lineage remains inspectable +6 model_free_verdict static validation and provenance only message delivery and reply lineage only cold resume, intent recovery, and acceptance evidence tie; correctness, traffic, cost, and duration remain live-run endpoints From 171c9e0862c8a3c9ad78e2f78edaa366ed4ddef0 Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 31 Jul 2026 22:38:03 +0200 Subject: [PATCH 4/6] Repin plan experiment to current PR115 head --- AGENT-SPEC.md | 4 ++-- cells/catalog-plan-vs-direct-brief/README.md | 10 ++++++++-- .../fixture/blockers.tsv | 2 +- .../fixture/check-fixtures.sh | 18 +++++++++--------- .../fixture/experiment.tsv | 4 ++-- .../fixture/provenance.tsv | 6 +++--- .../fixture/runner.tsv | 8 ++++---- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 4afa722f..f11eb7c4 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -251,8 +251,8 @@ This candidate contract is not part of the released `0fed14b` corpus conformance claim above. The paired model-free experiment is isolated to [`catalog-plan-vs-direct-brief`](cells/catalog-plan-vs-direct-brief/) and runs against st2 draft PR -[#115](https://github.com/compoundingtech/st2/pull/115) at exact source -`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`. +[#115](https://github.com/compoundingtech/st2/pull/115) at exact current-main source +`044964e4e07e3a656ccc1860bfff8517adf72c16`. The experimental plan surface is pinned to the [source sketch revision `5c1d142`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994). diff --git a/cells/catalog-plan-vs-direct-brief/README.md b/cells/catalog-plan-vs-direct-brief/README.md index 1124ad13..910c5c58 100644 --- a/cells/catalog-plan-vs-direct-brief/README.md +++ b/cells/catalog-plan-vs-direct-brief/README.md @@ -35,13 +35,19 @@ and neutral judge; it is not substituted for two agent executions. Arm A uses the exact external `plan.kdl` and agent `plan-ref` contract from [st2 draft PR #115](https://github.com/compoundingtech/st2/pull/115) at source -`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`, paired to the source gist at +`044964e4e07e3a656ccc1860bfff8517adf72c16`, paired to the source gist at revision `5c1d1427c0556d95d13890e5c5086cd85b25d994`. The accepted Linux artifact SHA256 is -`83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38`. +`38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81`. The fixture exercises only `plan validate`, `list`, `show`, and `inspect`, and proves they do not alter the catalog. +The current-main executable is not byte-identical to predecessor source +`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`; its binary SHA256 was +`83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38`. +The plan implementation and test blobs are byte-identical across the two +sources, as are this fixture's JSON outputs for all four read-only commands. + Arm B uses the product's ordinary durable message store rather than a synthetic thread sidecar. Neither arm receives a pre-authored acceptance receipt. The experiment uses plain copied folders, requires no server, and sets diff --git a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv index b2023fb8..2a333307 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv @@ -1,3 +1,3 @@ blocker_id owner dependency required_evidence status -st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact source 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff and binary SHA256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 with focused model 3/3, CLI 2/2, full workspace tests and doctests, scoped Clippy, rustfmt, diff receipts, and hosted run 30589548410 green ready +st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact current-main source 044964e4e07e3a656ccc1860bfff8517adf72c16 and binary SHA256 38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81 with full workspace tests, Clippy, rustfmt, and hosted run 30662043965 green ready live-ab evals-owner future provider-backed execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh index 2f0f37c0..a53e43fa 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh @@ -60,16 +60,16 @@ grep -Fqx $'steering_revision\t0001' "$experiment" grep -Fqx $'model_free_result\tno-measured-advantage-for-cold-resume-intent-recovery-or-acceptance-evidence' "$experiment" grep -Fqx $'source_gist\thttps://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" -grep -Fqx $'st2_plan_source\t2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$experiment" -grep -Fqx $'st2_plan_binary_sha256\t83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$experiment" +grep -Fqx $'st2_plan_source\t044964e4e07e3a656ccc1860bfff8517adf72c16' "$experiment" +grep -Fqx $'st2_plan_binary_sha256\t38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81' "$experiment" grep -Fqx $'live_run_status\tblocked-pending-separate-provider-authorization' "$experiment" grep -Fqx $'st2_pr\thttps://github.com/compoundingtech/st2/pull/115' "$runner" -grep -Fqx $'source_full\t2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$runner" -grep -Fqx $'source_short\t2caa0d7' "$runner" -grep -Fqx $'binary_sha256\t83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$runner" +grep -Fqx $'source_full\t044964e4e07e3a656ccc1860bfff8517adf72c16' "$runner" +grep -Fqx $'source_short\t044964e' "$runner" +grep -Fqx $'binary_sha256\t38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81' "$runner" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$runner" grep -Fqx $'runtime_scope\tread-only-validate-list-show-inspect' "$runner" -grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30589548410' "$runner" +grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30662043965' "$runner" grep -Fqx $'hosted_status\tpass' "$runner" cmp -s "$initial_plan_0000" "$plan_0000" @@ -201,9 +201,9 @@ echo "PROVENANCE-GREEN-43af" test "$(awk -F '\t' 'NR > 1 && $5 == "ready" { count++ } END { print count + 0 }' "$blockers")" -eq 1 test "$(awk -F '\t' 'NR > 1 && $5 == "blocked" { count++ } END { print count + 0 }' "$blockers")" -eq 1 -grep -Fq '2caa0d7f159c3c0d9c483bd63b2579d33f1986ff' "$blockers" -grep -Fq '83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38' "$blockers" -grep -Fq 'hosted run 30589548410 green' "$blockers" +grep -Fq '044964e4e07e3a656ccc1860bfff8517adf72c16' "$blockers" +grep -Fq '38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81' "$blockers" +grep -Fq 'hosted run 30662043965 green' "$blockers" grep -Fq 'separate authorization' "$blockers" test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 4 echo "PRODUCT-PAIRING-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv index e5fdc5a4..2c4fb548 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv @@ -3,8 +3,8 @@ experiment_id catalog-plan-vs-direct-brief-v1 source_gist https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994 source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 st2_plan_pr https://github.com/compoundingtech/st2/pull/115 -st2_plan_source 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff -st2_plan_binary_sha256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 +st2_plan_source 044964e4e07e3a656ccc1860bfff8517adf72c16 +st2_plan_binary_sha256 38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81 task_id receipt-report-latest-pass seed_repository task-repo allowed_change src/report.mjs diff --git a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv index 5cd290bf..7d529ecb 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv @@ -1,11 +1,11 @@ path sha256 -experiment.tsv d0cd5fe98e274c3e3ba5492d25553669e4f520c0170f61ee6fcdadd8a98ae48a +experiment.tsv 36ffc1fff60ed8e7aab88bb2a0d551936e004d57990407d5b6d14c2bd6628b60 arms.tsv 29572c02d123f4097aaf69b6fb8153d60190adecbcdcd75fdfad3690ca99df2f scenario.tsv 884a9c83b983128c8e642bf3a945d6f7edb544ac224f5934a4468f17a48d282b measurement-schema.tsv 01763a0c090b9c2fd39e5f1b7995380e2dc0d2e5d4230227749ddd41a133f9ab receipt-schema.tsv c996eaee81301640442b04d9fca31937bb840ed10ef78cca035e5990d1fe6441 -blockers.tsv 00b95bcb9bb3b308fda5c529072686e658dbcf9a6965eefd852679e823e30cc0 -runner.tsv 6e11100201af65c1b27e3b3540400497bb827f729b801aa870b21f1cd097bdbd +blockers.tsv f4c4d197935d0c5a0ffbfe19097dcd0f372f7d3131804178b61c31f528553480 +runner.tsv 8110f388ef265788994e02b25f865744ea30a2867ce1e55fe1d616af6cae578d arm-a/catalog-initial/plans/receipt-report/plan.kdl 3c883689f3709c7592c226c513af944a1036682612bfc7b0576dd1761de81a4a arm-a/catalog-initial/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e diff --git a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv index 0ed7de7a..02936199 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv @@ -1,9 +1,9 @@ field value st2_pr https://github.com/compoundingtech/st2/pull/115 -source_full 2caa0d7f159c3c0d9c483bd63b2579d33f1986ff -source_short 2caa0d7 -binary_sha256 83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38 +source_full 044964e4e07e3a656ccc1860bfff8517adf72c16 +source_short 044964e +binary_sha256 38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81 source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 runtime_scope read-only-validate-list-show-inspect -hosted_run https://github.com/compoundingtech/st2/actions/runs/30589548410 +hosted_run https://github.com/compoundingtech/st2/actions/runs/30662043965 hosted_status pass From a8e223f99c8f4b4e67e3390dec507fcbe992edf6 Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 31 Jul 2026 23:29:45 +0200 Subject: [PATCH 5/6] Correct plan history claims and repin PR115 --- AGENT-SPEC.md | 6 ++++-- cells/catalog-plan-vs-direct-brief/README.md | 16 +++++++++------- .../fixture/blockers.tsv | 2 +- .../fixture/check-fixtures.sh | 18 +++++++++--------- .../fixture/experiment.tsv | 4 ++-- .../fixture/provenance.tsv | 6 +++--- .../fixture/runner.tsv | 8 ++++---- 7 files changed, 32 insertions(+), 28 deletions(-) diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index f11eb7c4..86e37730 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -252,7 +252,7 @@ conformance claim above. The paired model-free experiment is isolated to [`catalog-plan-vs-direct-brief`](cells/catalog-plan-vs-direct-brief/) and runs against st2 draft PR [#115](https://github.com/compoundingtech/st2/pull/115) at exact current-main source -`044964e4e07e3a656ccc1860bfff8517adf72c16`. +`60d48bae5b7ac3a83c8d2c3324b61680bd6404dd`. The experimental plan surface is pinned to the [source sketch revision `5c1d142`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994). @@ -290,11 +290,13 @@ agent "app-web" { Plan identity is the explicit KDL value, never its directory. External plans require one `owner`. Each version requires one source-relative `file:` resource. -A version may repeat `parent` for multiple immutable parents; parents must +A version may repeat `parent` for multiple declared parents; parents must exist, be unique, and form an acyclic graph. A version with parents requires one non-empty `why`. The sorted frontier is derived as every version with no child, retaining concurrent siblings. `plan-ref` and version resources resolve relative to their declaring KDL and must remain inside the selected catalog. +The experiment stores no content digest or history. It cannot prove that an +earlier declaration or resource file stayed unchanged. The only supported CLI is read-only: diff --git a/cells/catalog-plan-vs-direct-brief/README.md b/cells/catalog-plan-vs-direct-brief/README.md index 910c5c58..73a3a6f4 100644 --- a/cells/catalog-plan-vs-direct-brief/README.md +++ b/cells/catalog-plan-vs-direct-brief/README.md @@ -35,18 +35,20 @@ and neutral judge; it is not substituted for two agent executions. Arm A uses the exact external `plan.kdl` and agent `plan-ref` contract from [st2 draft PR #115](https://github.com/compoundingtech/st2/pull/115) at source -`044964e4e07e3a656ccc1860bfff8517adf72c16`, paired to the source gist at +`60d48bae5b7ac3a83c8d2c3324b61680bd6404dd`, paired to the source gist at revision `5c1d1427c0556d95d13890e5c5086cd85b25d994`. The accepted Linux artifact SHA256 is -`38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81`. +`f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a`. The fixture exercises only `plan validate`, `list`, `show`, and `inspect`, and proves they do not alter the catalog. -The current-main executable is not byte-identical to predecessor source -`2caa0d7f159c3c0d9c483bd63b2579d33f1986ff`; its binary SHA256 was -`83efb0564a3cd366404495936b1e29a30ea210b0e429dbfeb6901830b2c49c38`. -The plan implementation and test blobs are byte-identical across the two -sources, as are this fixture's JSON outputs for all four read-only commands. +This plan model stores no content digest or history, so it cannot prove that an +earlier declaration, parent link, or resource file stayed unchanged. The +correction executable is not byte-identical to predecessor source +`044964e4e07e3a656ccc1860bfff8517adf72c16`; its binary SHA256 was +`38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81`. +The correction changes plan terminology only. The test blob and this fixture's +JSON outputs for all four read-only commands remain byte-identical. Arm B uses the product's ordinary durable message store rather than a synthetic thread sidecar. Neither arm receives a pre-authored acceptance receipt. The diff --git a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv index 2a333307..2bd632d5 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv @@ -1,3 +1,3 @@ blocker_id owner dependency required_evidence status -st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact current-main source 044964e4e07e3a656ccc1860bfff8517adf72c16 and binary SHA256 38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81 with full workspace tests, Clippy, rustfmt, and hosted run 30662043965 green ready +st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact current-main source 60d48bae5b7ac3a83c8d2c3324b61680bd6404dd and binary SHA256 f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a with full workspace tests, Clippy, rustfmt, and hosted run 30666101129 green ready live-ab evals-owner future provider-backed execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh index a53e43fa..7cec2ee5 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh @@ -60,16 +60,16 @@ grep -Fqx $'steering_revision\t0001' "$experiment" grep -Fqx $'model_free_result\tno-measured-advantage-for-cold-resume-intent-recovery-or-acceptance-evidence' "$experiment" grep -Fqx $'source_gist\thttps://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" -grep -Fqx $'st2_plan_source\t044964e4e07e3a656ccc1860bfff8517adf72c16' "$experiment" -grep -Fqx $'st2_plan_binary_sha256\t38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81' "$experiment" +grep -Fqx $'st2_plan_source\t60d48bae5b7ac3a83c8d2c3324b61680bd6404dd' "$experiment" +grep -Fqx $'st2_plan_binary_sha256\tf3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a' "$experiment" grep -Fqx $'live_run_status\tblocked-pending-separate-provider-authorization' "$experiment" grep -Fqx $'st2_pr\thttps://github.com/compoundingtech/st2/pull/115' "$runner" -grep -Fqx $'source_full\t044964e4e07e3a656ccc1860bfff8517adf72c16' "$runner" -grep -Fqx $'source_short\t044964e' "$runner" -grep -Fqx $'binary_sha256\t38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81' "$runner" +grep -Fqx $'source_full\t60d48bae5b7ac3a83c8d2c3324b61680bd6404dd' "$runner" +grep -Fqx $'source_short\t60d48ba' "$runner" +grep -Fqx $'binary_sha256\tf3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a' "$runner" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$runner" grep -Fqx $'runtime_scope\tread-only-validate-list-show-inspect' "$runner" -grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30662043965' "$runner" +grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30666101129' "$runner" grep -Fqx $'hosted_status\tpass' "$runner" cmp -s "$initial_plan_0000" "$plan_0000" @@ -201,9 +201,9 @@ echo "PROVENANCE-GREEN-43af" test "$(awk -F '\t' 'NR > 1 && $5 == "ready" { count++ } END { print count + 0 }' "$blockers")" -eq 1 test "$(awk -F '\t' 'NR > 1 && $5 == "blocked" { count++ } END { print count + 0 }' "$blockers")" -eq 1 -grep -Fq '044964e4e07e3a656ccc1860bfff8517adf72c16' "$blockers" -grep -Fq '38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81' "$blockers" -grep -Fq 'hosted run 30662043965 green' "$blockers" +grep -Fq '60d48bae5b7ac3a83c8d2c3324b61680bd6404dd' "$blockers" +grep -Fq 'f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a' "$blockers" +grep -Fq 'hosted run 30666101129 green' "$blockers" grep -Fq 'separate authorization' "$blockers" test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 4 echo "PRODUCT-PAIRING-GREEN-43af" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv index 2c4fb548..5190d77c 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv @@ -3,8 +3,8 @@ experiment_id catalog-plan-vs-direct-brief-v1 source_gist https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994 source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 st2_plan_pr https://github.com/compoundingtech/st2/pull/115 -st2_plan_source 044964e4e07e3a656ccc1860bfff8517adf72c16 -st2_plan_binary_sha256 38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81 +st2_plan_source 60d48bae5b7ac3a83c8d2c3324b61680bd6404dd +st2_plan_binary_sha256 f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a task_id receipt-report-latest-pass seed_repository task-repo allowed_change src/report.mjs diff --git a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv index 7d529ecb..4dcc1989 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv @@ -1,11 +1,11 @@ path sha256 -experiment.tsv 36ffc1fff60ed8e7aab88bb2a0d551936e004d57990407d5b6d14c2bd6628b60 +experiment.tsv 7c69cfb78860818d85bc8012ace75bb93bff7ca8604595f59f2861ca647c27f3 arms.tsv 29572c02d123f4097aaf69b6fb8153d60190adecbcdcd75fdfad3690ca99df2f scenario.tsv 884a9c83b983128c8e642bf3a945d6f7edb544ac224f5934a4468f17a48d282b measurement-schema.tsv 01763a0c090b9c2fd39e5f1b7995380e2dc0d2e5d4230227749ddd41a133f9ab receipt-schema.tsv c996eaee81301640442b04d9fca31937bb840ed10ef78cca035e5990d1fe6441 -blockers.tsv f4c4d197935d0c5a0ffbfe19097dcd0f372f7d3131804178b61c31f528553480 -runner.tsv 8110f388ef265788994e02b25f865744ea30a2867ce1e55fe1d616af6cae578d +blockers.tsv bfbe015226936b6101bfea898d04e7f6cbf7bd8441181b1f4b95495d2ed0fc4f +runner.tsv a28b1cfe8f264b13b782c920108f4c850ce2f5e6e54a14c53c41df9130350999 arm-a/catalog-initial/plans/receipt-report/plan.kdl 3c883689f3709c7592c226c513af944a1036682612bfc7b0576dd1761de81a4a arm-a/catalog-initial/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e diff --git a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv index 02936199..41ab04c7 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv @@ -1,9 +1,9 @@ field value st2_pr https://github.com/compoundingtech/st2/pull/115 -source_full 044964e4e07e3a656ccc1860bfff8517adf72c16 -source_short 044964e -binary_sha256 38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81 +source_full 60d48bae5b7ac3a83c8d2c3324b61680bd6404dd +source_short 60d48ba +binary_sha256 f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 runtime_scope read-only-validate-list-show-inspect -hosted_run https://github.com/compoundingtech/st2/actions/runs/30662043965 +hosted_run https://github.com/compoundingtech/st2/actions/runs/30666101129 hosted_status pass From 9fae14a192a1f11074de0ef2044e3235017bd476 Mon Sep 17 00:00:00 2001 From: Nathan Date: Mon, 3 Aug 2026 19:33:58 +0200 Subject: [PATCH 6/6] Update plan eval for Resource discovery --- AGENT-SPEC.md | 61 +++++++++++-------- CATALOG.md | 2 +- cells/catalog-plan-vs-direct-brief/README.md | 31 ++++++---- .../catalog-plan-vs-direct-brief.kdl | 7 ++- .../agents/eval/receipt-worker/agent.kdl | 3 +- .../plans/inline-intent/plan.kdl | 6 ++ .../plans/receipt-report/plan.kdl | 2 +- .../plans/receipt-report/versions/0000.md | 0 .../agents/eval/receipt-worker/agent.kdl | 3 +- .../plans/inline-intent/plan.kdl | 6 ++ .../plans/receipt-report/plan.kdl | 4 +- .../plans/receipt-report/versions/0000.md | 0 .../plans/receipt-report/versions/0001.md | 0 .../fixture/arms.tsv | 2 +- .../fixture/blockers.tsv | 2 +- .../fixture/check-fixtures.sh | 51 ++++++++++------ .../fixture/check-plan-contract.sh | 57 +++++++++++++---- .../fixture/experiment.tsv | 4 +- .../fixture/invalid/current/plan.kdl | 2 +- .../fixture/provenance.tsv | 30 ++++----- .../fixture/run-recovery-matrix.sh | 10 +-- .../fixture/runner.tsv | 8 +-- 22 files changed, 184 insertions(+), 107 deletions(-) create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/inline-intent/plan.kdl rename cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/{ => agents/eval/receipt-worker}/plans/receipt-report/plan.kdl (50%) rename cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/{ => agents/eval/receipt-worker}/plans/receipt-report/versions/0000.md (100%) create mode 100644 cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/inline-intent/plan.kdl rename cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/{ => agents/eval/receipt-worker}/plans/receipt-report/plan.kdl (59%) rename cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/{ => agents/eval/receipt-worker}/plans/receipt-report/versions/0000.md (100%) rename cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/{ => agents/eval/receipt-worker}/plans/receipt-report/versions/0001.md (100%) diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 86e37730..7284702f 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -252,51 +252,58 @@ conformance claim above. The paired model-free experiment is isolated to [`catalog-plan-vs-direct-brief`](cells/catalog-plan-vs-direct-brief/) and runs against st2 draft PR [#115](https://github.com/compoundingtech/st2/pull/115) at exact current-main source -`60d48bae5b7ac3a83c8d2c3324b61680bd6404dd`. +`8a76b6e71355140e5b89cd9313fcfd88c82b5cad`. -The experimental plan surface is pinned to the -[source sketch revision `5c1d142`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994). -An external plan is a top-level declaration: +The experiment shape descends from +[source sketch revision `5c1d142`](https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994); +the executable discovery and schema authority is the exact PR115 source above. +An agent discovers a plan only through a childless Resource link: ```kdl -plan "ship-remote-approvals" { - owner "app-web" - version "0000" resource="file:versions/0000.md" - version "0001" resource="file:versions/0001.md" { - parent "0000" - why "Browser proof exposed an approval race." - } +agent "app-web" { + resource "ship-remote-approvals" _tag="plan" uri="file:plans/ship-remote-approvals/plan.kdl" } ``` -An agent links it with a source-relative declaration: +The positional Resource name is an agent-local role. The referenced +`plan.kdl`, not the Resource envelope, owns all plan truth: ```kdl -agent "app-web" { - plan-ref "file:plans/ship-remote-approvals/plan.kdl" +plan "ship-remote-approvals" { + owner "app-web" + version "0000" content="file:versions/0000.md" + version "0001" content="file:versions/0001.md" { + parent "0000" + why "Browser proof exposed an approval race." + } } ``` -The equivalent agent-local form derives its owner from the containing explicit -agent identity: +Instead of external Markdown content, a version may keep its complete intent +inline in the same referenced `plan.kdl`: ```kdl -agent "app-web" { - plan "review-follow-up" { - version "0000" resource="file:plans/review-follow-up.md" +plan "review-follow-up" { + owner "app-web" + version "0000" { + intent "Review every unresolved comment and report the exact final head." } } ``` Plan identity is the explicit KDL value, never its directory. External plans -require one `owner`. Each version requires one source-relative `file:` resource. -A version may repeat `parent` for multiple declared parents; parents must -exist, be unique, and form an acyclic graph. A version with parents requires -one non-empty `why`. The sorted frontier is derived as every version with no -child, retaining concurrent siblings. `plan-ref` and version resources resolve -relative to their declaring KDL and must remain inside the selected catalog. -The experiment stores no content digest or history. It cannot prove that an -earlier declaration or resource file stayed unchanged. +require one `owner`. Each version requires exactly one `content="file:..."` or +one child `intent`; both or neither fail. A version may repeat `parent` for +multiple declared parents; parents must exist, be unique, and form an acyclic +graph. A version with parents requires one non-empty `why`. Versions are sorted, +and the frontier is derived as every version with no child, retaining concurrent +siblings. Resource URIs resolve relative to the agent KDL, while content URIs +resolve relative to the referenced `plan.kdl`; both must be relative `file:` +references to regular files inside the selected catalog. The Resource adds the +agent to `referencedBy` but owns no plan fields. Legacy `plan-ref`, childful plan +Resources, and agent-owned inline plan truth are unsupported. The experiment +stores no content digest or history. It cannot prove that an earlier declaration +or content file stayed unchanged. The only supported CLI is read-only: diff --git a/CATALOG.md b/CATALOG.md index 9afe5270..a1ceef12 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -27,7 +27,7 @@ harness-native loader plus canonical hook file. | `assignment-contract-hot-assignment` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `assignment-contract-hot-focus` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `assignment-contract-hot-resources` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | -| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 21 | — | **NO STRUCTURED RUN** | +| `catalog-plan-vs-direct-brief` | model-free | — | 0 | none | `60s` | 22 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | diff --git a/cells/catalog-plan-vs-direct-brief/README.md b/cells/catalog-plan-vs-direct-brief/README.md index 73a3a6f4..0ef697fe 100644 --- a/cells/catalog-plan-vs-direct-brief/README.md +++ b/cells/catalog-plan-vs-direct-brief/README.md @@ -33,22 +33,27 @@ Correctness, coordination traffic, token use, cost, and wall duration remain unresolved live-run endpoints. The reference solution establishes a valid task and neutral judge; it is not substituted for two agent executions. -Arm A uses the exact external `plan.kdl` and agent `plan-ref` contract from +Arm A uses the exact Resource-linked external `plan.kdl` contract from [st2 draft PR #115](https://github.com/compoundingtech/st2/pull/115) at source -`60d48bae5b7ac3a83c8d2c3324b61680bd6404dd`, paired to the source gist at -revision `5c1d1427c0556d95d13890e5c5086cd85b25d994`. The accepted Linux -artifact SHA256 is -`f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a`. -The fixture exercises only `plan validate`, `list`, `show`, and `inspect`, and -proves they do not alter the catalog. +`8a76b6e71355140e5b89cd9313fcfd88c82b5cad`, while retaining the experiment +shape from source gist revision +`5c1d1427c0556d95d13890e5c5086cd85b25d994`. A childless Agent Spec Resource +with `_tag="plan"` supplies only the agent-local role and source-relative file +link. The referenced `plan.kdl` owns the plan identity, owner, versions, and +intent truth. The comparison path keeps its external Markdown content, and one +focused second target proves the supported inline-intent form without changing +the A/B scenario. + +The accepted Linux artifact SHA256 is +`214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753`; +hosted Nix run +[30835684680](https://github.com/compoundingtech/st2/actions/runs/30835684680), +job `91760161352`, passed. The fixture exercises only `plan validate`, `list`, +`show`, and `inspect`, and proves they do not alter the catalog. Legacy +`plan-ref` and agent-owned inline plan truth are not used. This plan model stores no content digest or history, so it cannot prove that an -earlier declaration, parent link, or resource file stayed unchanged. The -correction executable is not byte-identical to predecessor source -`044964e4e07e3a656ccc1860bfff8517adf72c16`; its binary SHA256 was -`38a7fe4657e0583c34d13f73a9215e9f388e6cc2b87d6c9b9c70f07a9fce8f81`. -The correction changes plan terminology only. The test blob and this fixture's -JSON outputs for all four read-only commands remain byte-identical. +earlier declaration, parent link, or content file stayed unchanged. Arm B uses the product's ordinary durable message store rather than a synthetic thread sidecar. Neither arm receives a pre-authored acceptance receipt. The diff --git a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl index 8bf879f2..e48d310d 100644 --- a/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl +++ b/cells/catalog-plan-vs-direct-brief/catalog-plan-vs-direct-brief.kdl @@ -56,16 +56,19 @@ eval { judge "RUNNER — the exact experimental st2 binary matches the accepted source and SHA256" { exec "grep -Fqx PLAN-RUNNER-PIN-GREEN-43af $RUNS_DIR/plan-contract.out" } - judge "PLAN VALIDATE — the external plan and agent reference normalize without errors" { + judge "PLAN VALIDATE — Resource-linked external plans normalize without errors" { exec "grep -Fqx PLAN-VALIDATE-GREEN-43af $RUNS_DIR/plan-contract.out" } + judge "PLAN TARGETS — external Markdown content and inline intent are both exact" { + exec "grep -Fqx PLAN-TARGET-FORMS-GREEN-43af $RUNS_DIR/plan-contract.out" + } judge "PLAN LIST — explicit identity, owner, and derived frontier are exact" { exec "grep -Fqx PLAN-LIST-GREEN-43af $RUNS_DIR/plan-contract.out" } judge "PLAN SHOW — normalized intent exposes versions but not provenance paths" { exec "grep -Fqx PLAN-SHOW-GREEN-43af $RUNS_DIR/plan-contract.out" } - judge "PLAN INSPECT — provenance and resolved resources are exact and read-only" { + judge "PLAN INSPECT — provenance and resolved content are exact and read-only" { exec "grep -Fqx PLAN-INSPECT-READONLY-GREEN-43af $RUNS_DIR/plan-contract.out" } judge "PLAN BOUNDARY — a mutable current pointer is rejected with a classified error" { diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl index dacd0fe6..66133e08 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl @@ -1,5 +1,6 @@ agent "receipt-worker" { host "eval" command "true" - plan-ref "file:../../../plans/receipt-report/plan.kdl" + resource "receipt-report" _tag="plan" uri="file:plans/receipt-report/plan.kdl" + resource "inline-intent" _tag="plan" uri="file:plans/inline-intent/plan.kdl" } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/inline-intent/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/inline-intent/plan.kdl new file mode 100644 index 00000000..7e37f031 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/inline-intent/plan.kdl @@ -0,0 +1,6 @@ +plan "inline-intent" { + owner "receipt-worker" + version "0000" { + intent "Keep the complete inline intent in plan.kdl." + } +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/plan.kdl similarity index 50% rename from cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl rename to cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/plan.kdl index d1d055e4..63844e95 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/plan.kdl +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/plan.kdl @@ -1,4 +1,4 @@ plan "receipt-report" { owner "receipt-worker" - version "0000" resource="file:versions/0000.md" + version "0000" content="file:versions/0000.md" } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/versions/0000.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md similarity index 100% rename from cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/plans/receipt-report/versions/0000.md rename to cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl index dacd0fe6..66133e08 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/agent.kdl @@ -1,5 +1,6 @@ agent "receipt-worker" { host "eval" command "true" - plan-ref "file:../../../plans/receipt-report/plan.kdl" + resource "receipt-report" _tag="plan" uri="file:plans/receipt-report/plan.kdl" + resource "inline-intent" _tag="plan" uri="file:plans/inline-intent/plan.kdl" } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/inline-intent/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/inline-intent/plan.kdl new file mode 100644 index 00000000..7e37f031 --- /dev/null +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/inline-intent/plan.kdl @@ -0,0 +1,6 @@ +plan "inline-intent" { + owner "receipt-worker" + version "0000" { + intent "Keep the complete inline intent in plan.kdl." + } +} diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/plan.kdl similarity index 59% rename from cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl rename to cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/plan.kdl index 7f7f5931..c556d303 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/plan.kdl +++ b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/plan.kdl @@ -1,8 +1,8 @@ plan "receipt-report" { owner "receipt-worker" - version "0000" resource="file:versions/0000.md" - version "0001" resource="file:versions/0001.md" { + version "0000" content="file:versions/0000.md" + version "0001" content="file:versions/0001.md" { parent "0000" why "Human steering tightens input validation without changing scope." } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0000.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md similarity index 100% rename from cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0000.md rename to cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0001.md b/cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md similarity index 100% rename from cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/plans/receipt-report/versions/0001.md rename to cells/catalog-plan-vs-direct-brief/fixture/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md diff --git a/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv b/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv index a7111672..c67b5f43 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/arms.tsv @@ -1,3 +1,3 @@ arm input_kind initial_source local_resume_source steering_source acceptance_surface allowed_tools wall_budget_seconds model_token_budget judge_profile done_condition -A versioned-catalog-plan arm-a/catalog-initial/plans/receipt-report/versions/0000.md local plan catalog inspected through st2 plan arm-a/catalog/plans/receipt-report/versions/0001.md evaluator-owned receipt only node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change +A versioned-catalog-plan arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md local plan catalog inspected through st2 plan arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md evaluator-owned receipt only node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change B durable-direct-brief arm-b/inbox/brief-0000.md local st2 message inbox read through st2 message arm-b/inbox/brief-0001.md evaluator-owned receipt only node,bash,git 1200 40000 arm-neutral-public-plus-held-out public and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change diff --git a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv index 2bd632d5..c5b3c698 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/blockers.tsv @@ -1,3 +1,3 @@ blocker_id owner dependency required_evidence status -st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact current-main source 60d48bae5b7ac3a83c8d2c3324b61680bd6404dd and binary SHA256 f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a with full workspace tests, Clippy, rustfmt, and hosted run 30666101129 green ready +st2-plan-runtime st2-owner https://github.com/compoundingtech/st2/pull/115 exact Resource-link source 8a76b6e71355140e5b89cd9313fcfd88c82b5cad and binary SHA256 214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753 with prior gates and hosted run 30835684680 job 91760161352 green ready live-ab evals-owner future provider-backed execution separate authorization naming exact plan artifact, model, effort, per-arm token and wall budgets, arm order or counterbalancing, maximum spend, cleanup, and rollback blocked diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh index 7cec2ee5..42c338f3 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-fixtures.sh @@ -9,13 +9,15 @@ measurements="$root/measurement-schema.tsv" receipts="$root/receipt-schema.tsv" blockers="$root/blockers.tsv" runner="$root/runner.tsv" -plan_declaration="$root/arm-a/catalog/plans/receipt-report/plan.kdl" +plan_declaration="$root/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/plan.kdl" agent_declaration="$root/arm-a/catalog/agents/eval/receipt-worker/agent.kdl" -initial_plan_declaration="$root/arm-a/catalog-initial/plans/receipt-report/plan.kdl" +inline_plan_declaration="$root/arm-a/catalog/agents/eval/receipt-worker/plans/inline-intent/plan.kdl" +initial_plan_declaration="$root/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/plan.kdl" initial_agent_declaration="$root/arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl" -initial_plan_0000="$root/arm-a/catalog-initial/plans/receipt-report/versions/0000.md" -plan_0000="$root/arm-a/catalog/plans/receipt-report/versions/0000.md" -plan_0001="$root/arm-a/catalog/plans/receipt-report/versions/0001.md" +initial_inline_plan_declaration="$root/arm-a/catalog-initial/agents/eval/receipt-worker/plans/inline-intent/plan.kdl" +initial_plan_0000="$root/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md" +plan_0000="$root/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md" +plan_0001="$root/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md" brief_0000="$root/arm-b/inbox/brief-0000.md" brief_0001="$root/arm-b/inbox/brief-0001.md" @@ -60,16 +62,16 @@ grep -Fqx $'steering_revision\t0001' "$experiment" grep -Fqx $'model_free_result\tno-measured-advantage-for-cold-resume-intent-recovery-or-acceptance-evidence' "$experiment" grep -Fqx $'source_gist\thttps://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$experiment" -grep -Fqx $'st2_plan_source\t60d48bae5b7ac3a83c8d2c3324b61680bd6404dd' "$experiment" -grep -Fqx $'st2_plan_binary_sha256\tf3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a' "$experiment" +grep -Fqx $'st2_plan_source\t8a76b6e71355140e5b89cd9313fcfd88c82b5cad' "$experiment" +grep -Fqx $'st2_plan_binary_sha256\t214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753' "$experiment" grep -Fqx $'live_run_status\tblocked-pending-separate-provider-authorization' "$experiment" grep -Fqx $'st2_pr\thttps://github.com/compoundingtech/st2/pull/115' "$runner" -grep -Fqx $'source_full\t60d48bae5b7ac3a83c8d2c3324b61680bd6404dd' "$runner" -grep -Fqx $'source_short\t60d48ba' "$runner" -grep -Fqx $'binary_sha256\tf3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a' "$runner" +grep -Fqx $'source_full\t8a76b6e71355140e5b89cd9313fcfd88c82b5cad' "$runner" +grep -Fqx $'source_short\t8a76b6e' "$runner" +grep -Fqx $'binary_sha256\t214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753' "$runner" grep -Fqx $'source_gist_revision\t5c1d1427c0556d95d13890e5c5086cd85b25d994' "$runner" grep -Fqx $'runtime_scope\tread-only-validate-list-show-inspect' "$runner" -grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30666101129' "$runner" +grep -Fqx $'hosted_run\thttps://github.com/compoundingtech/st2/actions/runs/30835684680' "$runner" grep -Fqx $'hosted_status\tpass' "$runner" cmp -s "$initial_plan_0000" "$plan_0000" @@ -93,25 +95,34 @@ awk -F '\t' ' } } ' "$arms" -grep -Fqx $'A\tversioned-catalog-plan\tarm-a/catalog-initial/plans/receipt-report/versions/0000.md\tlocal plan catalog inspected through st2 plan\tarm-a/catalog/plans/receipt-report/versions/0001.md\tevaluator-owned receipt only\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" +grep -Fqx $'A\tversioned-catalog-plan\tarm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md\tlocal plan catalog inspected through st2 plan\tarm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md\tevaluator-owned receipt only\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" grep -Fqx $'B\tdurable-direct-brief\tarm-b/inbox/brief-0000.md\tlocal st2 message inbox read through st2 message\tarm-b/inbox/brief-0001.md\tevaluator-owned receipt only\tnode,bash,git\t1200\t40000\tarm-neutral-public-plus-held-out\tpublic and held-out tests pass; only src/report.mjs differs from the seed; no dependency or network change' "$arms" grep -Fqx 'plan "receipt-report" {' "$plan_declaration" grep -Fqx ' owner "receipt-worker"' "$plan_declaration" -grep -Fqx ' version "0000" resource="file:versions/0000.md"' "$plan_declaration" -grep -Fqx ' version "0001" resource="file:versions/0001.md" {' "$plan_declaration" +grep -Fqx ' version "0000" content="file:versions/0000.md"' "$plan_declaration" +grep -Fqx ' version "0001" content="file:versions/0001.md" {' "$plan_declaration" grep -Fqx ' parent "0000"' "$plan_declaration" grep -Fqx ' why "Human steering tightens input validation without changing scope."' "$plan_declaration" grep -Fqx 'plan "receipt-report" {' "$initial_plan_declaration" grep -Fqx ' owner "receipt-worker"' "$initial_plan_declaration" -grep -Fqx ' version "0000" resource="file:versions/0000.md"' "$initial_plan_declaration" +grep -Fqx ' version "0000" content="file:versions/0000.md"' "$initial_plan_declaration" if grep -Fq 'version "0001"' "$initial_plan_declaration"; then echo "initial plan snapshot already contains the steering revision" >&2 exit 1 fi grep -Fqx 'agent "receipt-worker" {' "$agent_declaration" -grep -Fqx ' plan-ref "file:../../../plans/receipt-report/plan.kdl"' "$agent_declaration" +grep -Fqx ' resource "receipt-report" _tag="plan" uri="file:plans/receipt-report/plan.kdl"' "$agent_declaration" +grep -Fqx ' resource "inline-intent" _tag="plan" uri="file:plans/inline-intent/plan.kdl"' "$agent_declaration" +grep -Fqx 'plan "inline-intent" {' "$inline_plan_declaration" +grep -Fqx ' owner "receipt-worker"' "$inline_plan_declaration" +grep -Fqx ' intent "Keep the complete inline intent in plan.kdl."' "$inline_plan_declaration" +cmp -s "$initial_inline_plan_declaration" "$inline_plan_declaration" cmp -s "$initial_agent_declaration" "$agent_declaration" +if rg -n 'plan-ref|^[[:space:]]+plan[[:space:]]' "$initial_agent_declaration" "$agent_declaration"; then + echo "agent declaration still owns plan truth" >&2 + exit 1 +fi if ( cd "$root/task-repo" @@ -201,11 +212,11 @@ echo "PROVENANCE-GREEN-43af" test "$(awk -F '\t' 'NR > 1 && $5 == "ready" { count++ } END { print count + 0 }' "$blockers")" -eq 1 test "$(awk -F '\t' 'NR > 1 && $5 == "blocked" { count++ } END { print count + 0 }' "$blockers")" -eq 1 -grep -Fq '60d48bae5b7ac3a83c8d2c3324b61680bd6404dd' "$blockers" -grep -Fq 'f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a' "$blockers" -grep -Fq 'hosted run 30666101129 green' "$blockers" +grep -Fq '8a76b6e71355140e5b89cd9313fcfd88c82b5cad' "$blockers" +grep -Fq '214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753' "$blockers" +grep -Fq 'hosted run 30835684680 job 91760161352 green' "$blockers" grep -Fq 'separate authorization' "$blockers" -test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 4 +test "$(find "$root/arm-a" -type f -name '*.kdl' | wc -l)" -eq 6 echo "PRODUCT-PAIRING-GREEN-43af" if rg -n --pcre2 \ diff --git a/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh b/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh index 5a115afd..cb08fea4 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/check-plan-contract.sh @@ -35,17 +35,24 @@ find "$catalog" -type f -print0 | validate_json="$("$st2_path" plan validate --catalog "$catalog" --json)" jq -e ' .result == "valid" and - .plans == 1 and + .plans == 2 and .errors == 0 ' <<<"$validate_json" >/dev/null echo "PLAN-VALIDATE-GREEN-43af" list_json="$("$st2_path" plan list --catalog "$catalog" --json)" jq -e ' - length == 1 and - .[0].identity == "receipt-report" and - .[0].owner == "receipt-worker" and - .[0].frontier == ["0001"] + length == 2 and + .[0] == { + "identity": "inline-intent", + "owner": "receipt-worker", + "frontier": ["0000"] + } and + .[1] == { + "identity": "receipt-report", + "owner": "receipt-worker", + "frontier": ["0001"] + } ' <<<"$list_json" >/dev/null echo "PLAN-LIST-GREEN-43af" @@ -59,16 +66,16 @@ jq -e ' "identity": "0000", "parents": [], "why": null, - "resource": "file:versions/0000.md" + "content": "file:versions/0000.md" } and .versions[1] == { "identity": "0001", "parents": ["0000"], "why": "Human steering tightens input validation without changing scope.", - "resource": "file:versions/0001.md" + "content": "file:versions/0001.md" } and (has("source") | not) and - (.versions[0] | has("resolvedResource") | not) + (.versions[0] | has("resolvedContent") | not) ' <<<"$show_json" >/dev/null echo "PLAN-SHOW-GREEN-43af" @@ -79,11 +86,39 @@ jq -e ' .sourceKind == "external" and .referencedBy == ["receipt-worker"] and .frontier == ["0001"] and - (.source | endswith("/plans/receipt-report/plan.kdl")) and - (.versions[0].resolvedResource | endswith("/plans/receipt-report/versions/0000.md")) and - (.versions[1].resolvedResource | endswith("/plans/receipt-report/versions/0001.md")) + (.source | endswith("/agents/eval/receipt-worker/plans/receipt-report/plan.kdl")) and + (.versions[0].resolvedContent | endswith("/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md")) and + (.versions[1].resolvedContent | endswith("/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md")) ' <<<"$inspect_json" >/dev/null +inline_show_json="$("$st2_path" plan show inline-intent --catalog "$catalog" --json)" +jq -e ' + .identity == "inline-intent" and + .owner == "receipt-worker" and + .frontier == ["0000"] and + .versions == [{ + "identity": "0000", + "parents": [], + "why": null, + "intent": "Keep the complete inline intent in plan.kdl." + }] and + (has("source") | not) and + (.versions[0] | has("content") | not) and + (.versions[0] | has("resolvedContent") | not) +' <<<"$inline_show_json" >/dev/null + +inline_inspect_json="$("$st2_path" plan inspect inline-intent --catalog "$catalog" --json)" +jq -e ' + .identity == "inline-intent" and + .sourceKind == "external" and + .referencedBy == ["receipt-worker"] and + (.source | endswith("/agents/eval/receipt-worker/plans/inline-intent/plan.kdl")) and + .versions[0].intent == "Keep the complete inline intent in plan.kdl." and + (.versions[0] | has("content") | not) and + (.versions[0] | has("resolvedContent") | not) +' <<<"$inline_inspect_json" >/dev/null +echo "PLAN-TARGET-FORMS-GREEN-43af" + find "$catalog" -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum >"$after" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv index 5190d77c..e5eff76a 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/experiment.tsv @@ -3,8 +3,8 @@ experiment_id catalog-plan-vs-direct-brief-v1 source_gist https://gist.github.com/myobie/d5ecfac24cd3965e095a5031cd2e00cb/5c1d1427c0556d95d13890e5c5086cd85b25d994 source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 st2_plan_pr https://github.com/compoundingtech/st2/pull/115 -st2_plan_source 60d48bae5b7ac3a83c8d2c3324b61680bd6404dd -st2_plan_binary_sha256 f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a +st2_plan_source 8a76b6e71355140e5b89cd9313fcfd88c82b5cad +st2_plan_binary_sha256 214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753 task_id receipt-report-latest-pass seed_repository task-repo allowed_change src/report.mjs diff --git a/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl b/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl index 8b31bac7..999c07d3 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl +++ b/cells/catalog-plan-vs-direct-brief/fixture/invalid/current/plan.kdl @@ -1,5 +1,5 @@ plan "unsupported-current" { owner "receipt-worker" current "0000" - version "0000" resource="file:0000.md" + version "0000" content="file:0000.md" } diff --git a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv index 4dcc1989..0a0466c8 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/provenance.tsv @@ -1,21 +1,23 @@ path sha256 -experiment.tsv 7c69cfb78860818d85bc8012ace75bb93bff7ca8604595f59f2861ca647c27f3 -arms.tsv 29572c02d123f4097aaf69b6fb8153d60190adecbcdcd75fdfad3690ca99df2f +experiment.tsv 7eb1c81bb4aa26c37aed62488602fc8ddda83fe946301832682252c440bc605e +arms.tsv bd7c41b75fc29920b5ba1d1298649f5b94e65cf85ed9b8192def51e22357e7b2 scenario.tsv 884a9c83b983128c8e642bf3a945d6f7edb544ac224f5934a4468f17a48d282b measurement-schema.tsv 01763a0c090b9c2fd39e5f1b7995380e2dc0d2e5d4230227749ddd41a133f9ab receipt-schema.tsv c996eaee81301640442b04d9fca31937bb840ed10ef78cca035e5990d1fe6441 -blockers.tsv bfbe015226936b6101bfea898d04e7f6cbf7bd8441181b1f4b95495d2ed0fc4f -runner.tsv a28b1cfe8f264b13b782c920108f4c850ce2f5e6e54a14c53c41df9130350999 -arm-a/catalog-initial/plans/receipt-report/plan.kdl 3c883689f3709c7592c226c513af944a1036682612bfc7b0576dd1761de81a4a -arm-a/catalog-initial/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 -arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e -arm-a/catalog/plans/receipt-report/plan.kdl 1d5f559823f228896cce5e018c0745eca458267af636c94044de6a27f8f62029 -arm-a/catalog/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 -arm-a/catalog/plans/receipt-report/versions/0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 -arm-a/catalog/agents/eval/receipt-worker/agent.kdl 132c6eec21704a6e6d02091e42c8f6e8262bdd08a9b2916a0493655ed7ab970e +blockers.tsv 140a4fb38bae58ef5dbace2a10574000808d3d1fd5b8e609b829850dc5fb49ba +runner.tsv 8114588d818d2306d944785953447cb6114bdb83e0e72c6ebf3bc060016b19b8 +arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/plan.kdl 08dca1863eadce1d1616d29d1f01de21435b07015d36066f42e5dbe8d68edbd1 +arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +arm-a/catalog-initial/agents/eval/receipt-worker/plans/inline-intent/plan.kdl c1cf3ed40cc909edb8bcb69418dcbe21daf2ed074be34638e632cc31c4f0ae23 +arm-a/catalog-initial/agents/eval/receipt-worker/agent.kdl e347308269970ab6abde6c1f99a96e0619783f55081ec348d9107c10a10a4e92 +arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/plan.kdl 9781d56aceee726cf26c639b195d100fb66229b0855fa9ba5ce2f1e56024e9ba +arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 +arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 +arm-a/catalog/agents/eval/receipt-worker/plans/inline-intent/plan.kdl c1cf3ed40cc909edb8bcb69418dcbe21daf2ed074be34638e632cc31c4f0ae23 +arm-a/catalog/agents/eval/receipt-worker/agent.kdl e347308269970ab6abde6c1f99a96e0619783f55081ec348d9107c10a10a4e92 arm-b/inbox/brief-0000.md 4f81d370fe6c09c5c199046cc948f68ebdc8a510a4c30fde8edc6993d1abef11 arm-b/inbox/brief-0001.md aa3075e6fcbec4648209f77cd0b3b9866d70f4a354bb16554d3e58babd7f04b4 -invalid/current/plan.kdl f6ac18a143e0a6ffb060a003208c64fd59dd52e7a90f5613de23b1aa82ce7a34 +invalid/current/plan.kdl a7d5792410bcd4f6ba073a1fecaee536adec7a40f82298c2d3022a75f9a0b687 invalid/current/0000.md 1a18971d45c564ee22de2496c1a086b6a799b9cf809a6d7351236d3c5b4e788a task-repo/package.json 8f756153985ab193ec21992e4adf874db4b02330db019006c5214550c2ff9618 task-repo/README.md c50a603eac8bb3dad994dfc96f682f7f8d9a56300e21a7f7be0d040308ff580b @@ -24,6 +26,6 @@ task-repo/test/report.test.mjs e9008004b55687df44025f66abe2e5ff739de8d8c7bec27d8 reference/report.mjs 7343d91050754693f1ced37a97dae6f8701549af27ee78e08bab260a5d6550c9 judges/report.test.mjs 24f41a2e596c940a8a3c5123e53b635bc66deb11d48bc0911a280d7a5485e670 judges/judge-task.sh 23680852e5d3ffd744eb00b6edf12c85f5560cbe3b406da45169e710eff2a10b -run-recovery-matrix.sh 8d1ddf23ecde6ba0d0ed763b71f690b962c66e0e80f72d35b0580e27dc5c1c87 -check-plan-contract.sh a1b225718ac2f9c9c4e7c5522398273d9cee081255fa39bcedbcdaeef36d8c17 +run-recovery-matrix.sh 29aab23eeca0c1af1f76c6745da11c37da4e8a1800592a8a53df4eb963941533 +check-plan-contract.sh 474a5b9ffa62d899ed81f4867f245202e6c161593d2e7cec5c2bcdfb969f5335 task-repo.tree 22c368e3d2b9722c0fe53710dbd16e2010e98d7215db0aa42fe7532cfa9233c5 diff --git a/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh b/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh index ef9398c5..fc842384 100755 --- a/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh +++ b/cells/catalog-plan-vs-direct-brief/fixture/run-recovery-matrix.sh @@ -44,7 +44,7 @@ local_store="$scratch/local" session="$scratch/session" plan_local="$local_store/plan-catalog" direct_bus="$local_store/direct-bus" -mkdir -p "$remote" "$local_store" "$session" +mkdir -p "$remote" "$local_store" "$session" "$direct_bus" cp -a "$root/arm-a/catalog-initial" "$remote/plan-initial" cp -a "$root/arm-a/catalog" "$remote/plan-steered" cp -a "$root/arm-b/inbox" "$remote/direct-source" @@ -73,9 +73,9 @@ jq -e ' (.versions | length) == 1 and .versions[0].identity == "0000" ' <<<"$plan_initial_show" >/dev/null -plan_initial_path="$(jq -r '.versions[0].resolvedResource' <<<"$plan_initial_inspect")" +plan_initial_path="$(jq -r '.versions[0].resolvedContent' <<<"$plan_initial_inspect")" plan_initial_sha="$(hash_file "$plan_initial_path")" -test "$plan_initial_sha" = "$(hash_file "$root/arm-a/catalog-initial/plans/receipt-report/versions/0000.md")" +test "$plan_initial_sha" = "$(hash_file "$root/arm-a/catalog-initial/agents/eval/receipt-worker/plans/receipt-report/versions/0000.md")" assert_no_acceptance_claim <<<"$plan_initial_show" assert_no_acceptance_claim <<<"$plan_initial_inspect" @@ -123,9 +123,9 @@ jq -e ' .versions[1].identity == "0001" and .versions[1].parents == ["0000"] ' <<<"$plan_steered_show" >/dev/null -plan_steered_path="$(jq -r '.versions[] | select(.identity == "0001") | .resolvedResource' <<<"$plan_steered_inspect")" +plan_steered_path="$(jq -r '.versions[] | select(.identity == "0001") | .resolvedContent' <<<"$plan_steered_inspect")" plan_steered_sha="$(hash_file "$plan_steered_path")" -test "$plan_steered_sha" = "$(hash_file "$root/arm-a/catalog/plans/receipt-report/versions/0001.md")" +test "$plan_steered_sha" = "$(hash_file "$root/arm-a/catalog/agents/eval/receipt-worker/plans/receipt-report/versions/0001.md")" assert_no_acceptance_claim <<<"$plan_steered_show" assert_no_acceptance_claim <<<"$plan_steered_inspect" diff --git a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv index 41ab04c7..2a807f59 100644 --- a/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv +++ b/cells/catalog-plan-vs-direct-brief/fixture/runner.tsv @@ -1,9 +1,9 @@ field value st2_pr https://github.com/compoundingtech/st2/pull/115 -source_full 60d48bae5b7ac3a83c8d2c3324b61680bd6404dd -source_short 60d48ba -binary_sha256 f3e935ee8e6c38b5ba29f0507b4639fe2212e3764b43e18967f6762a5962e48a +source_full 8a76b6e71355140e5b89cd9313fcfd88c82b5cad +source_short 8a76b6e +binary_sha256 214e08874720bc546d4adf7d7977e614237baf7989cc09f6932cd991f497a753 source_gist_revision 5c1d1427c0556d95d13890e5c5086cd85b25d994 runtime_scope read-only-validate-list-show-inspect -hosted_run https://github.com/compoundingtech/st2/actions/runs/30666101129 +hosted_run https://github.com/compoundingtech/st2/actions/runs/30835684680 hosted_status pass