From 6771d0b6ebb99aa3bd478731244bb3a9f9e36d60 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:05:08 +0200 Subject: [PATCH 01/12] evals: cover agent-new interview handoff agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- CATALOG.md | 9 +- cells/agent-new-bundle-contract/README.md | 23 +++++ .../agent-new-bundle-contract.kdl | 39 +++++++++ .../fixture/inputs/forbidden-account.json | 19 ++++ .../fixture/inputs/implementation.json | 18 ++++ .../fixture/inputs/review.json | 18 ++++ .../fixture/inputs/trajectory-drift.json | 18 ++++ .../fixture/render-intent.sh | 86 +++++++++++++++++++ .../agent-new-bundle-contract/judges/grade.sh | 46 ++++++++++ .../judges/mutation-check.sh | 32 +++++++ cells/agent-new-interview/README.md | 27 ++++++ .../agent-new-interview.kdl | 36 ++++++++ .../fixture/interviewer/CLAUDE.md | 1 + .../fixture/interviewer/PERSONA.md | 48 +++++++++++ .../interviewer/references/issue-40.md | 8 ++ .../fixture/interviewer/render-intent.sh | 74 ++++++++++++++++ .../fixture/interviewer/submit-intent | 8 ++ cells/agent-new-interview/judges/grade.sh | 54 ++++++++++++ cells/agent-new-interview/task.md | 1 + docs/vrs/spec.md | 29 +++++++ evidence/harness-exclusions.tsv | 1 + 21 files changed, 592 insertions(+), 3 deletions(-) create mode 100644 cells/agent-new-bundle-contract/README.md create mode 100644 cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl create mode 100644 cells/agent-new-bundle-contract/fixture/inputs/forbidden-account.json create mode 100644 cells/agent-new-bundle-contract/fixture/inputs/implementation.json create mode 100644 cells/agent-new-bundle-contract/fixture/inputs/review.json create mode 100644 cells/agent-new-bundle-contract/fixture/inputs/trajectory-drift.json create mode 100755 cells/agent-new-bundle-contract/fixture/render-intent.sh create mode 100755 cells/agent-new-bundle-contract/judges/grade.sh create mode 100755 cells/agent-new-bundle-contract/judges/mutation-check.sh create mode 100644 cells/agent-new-interview/README.md create mode 100644 cells/agent-new-interview/agent-new-interview.kdl create mode 100644 cells/agent-new-interview/fixture/interviewer/CLAUDE.md create mode 100644 cells/agent-new-interview/fixture/interviewer/PERSONA.md create mode 100644 cells/agent-new-interview/fixture/interviewer/references/issue-40.md create mode 100755 cells/agent-new-interview/fixture/interviewer/render-intent.sh create mode 100755 cells/agent-new-interview/fixture/interviewer/submit-intent create mode 100755 cells/agent-new-interview/judges/grade.sh create mode 100644 cells/agent-new-interview/task.md diff --git a/CATALOG.md b/CATALOG.md index 58cd2eb1..87ca4867 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,8 +1,8 @@ # Current eval catalog and evidence -This is the sole current corpus surface: **45 included cells** (14 model-free, -31 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active +This is the sole current corpus surface: **47 included cells** (15 model-free, +32 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active KDL and executable judge scripts. Latest accepted PASS evidence remains distinct from the append-only run history: a failed last run is visible without being advertised as accepted evidence, and cells with no structured run row say so explicitly. @@ -16,6 +16,8 @@ harness-native loader plus canonical hook file. | Cell | Harness | Model(s) / effort | Model seats | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | |---|---|---|---:|---|---|---:|---|---| +| `agent-new-bundle-contract` | model-free | — | 0 | none | `60s` | 5 | — | **NO STRUCTURED RUN** | +| `agent-new-interview` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | @@ -81,6 +83,7 @@ while the last-run column makes a recorded failure distinct from a cell with no ## Harness hook exclusions +- `agent-new-bundle-contract` / `cell` — deterministic semantic-intent lowering and mutation probe; no harness seat - `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness seat - `docs` / `judge:cold-reader` — one-shot offline Claude print grader; no bus identity, DING, or hook surface - `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness seat @@ -97,7 +100,7 @@ while the last-run column makes a recorded failure distinct from a cell with no - `targeted-reconcile-isolation` / `cell` — deterministic selected reconcile isolation probe; no bus-connected agent - `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness seat -These **15** rows are generated from structured exclusions. A cell-level row is required for +These **16** rows are generated from structured exclusions. A cell-level row is required for every derived model-free cell. The one-shot docs cold reader is a model-backed grader, but not a bus-connected agent: it has no st2 identity, DING sidecar, or lifecycle hooks. diff --git a/cells/agent-new-bundle-contract/README.md b/cells/agent-new-bundle-contract/README.md new file mode 100644 index 00000000..bd0adaa1 --- /dev/null +++ b/cells/agent-new-bundle-contract/README.md @@ -0,0 +1,23 @@ +# agent-new-bundle-contract + +This model-free cell freezes the deterministic half of the `axe agent new` +interview handoff. A small semantic `axe.agent-creation-intent.v1` record is +the only interviewer output. The renderer owns canonical Agent Spec KDL and +the initial inbox Resource. + +Two valid one-session intents exercise the same lowering path. Two invalid +inputs prove the input boundary is closed: an interviewer cannot pin an +account or introduce an unsupported trajectory value. + +Held-out mutation checks copy valid output and independently prove that the +bundle grader rejects: + +- a bare provider argv in place of `axe agent launch`; +- drift between typed intent and the KDL trajectory; and +- a missing initial inbox Resource. + +Run it with: + +```sh +st2 eval ./cells/agent-new-bundle-contract/ +``` diff --git a/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl b/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl new file mode 100644 index 00000000..392e71e6 --- /dev/null +++ b/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl @@ -0,0 +1,39 @@ +// Deterministic B2 lowering: typed creation intent becomes one canonical Agent Spec +// plus the initial inbox Resource, while invalid or over-specified intent fails closed. +eval { + copy "./fixture" + max-timeout "60s" + + run "implementation" { + command "bash ./render-intent.sh ./inputs/implementation.json ./out/implementation" + } + run "review" { + command "bash ./render-intent.sh ./inputs/review.json ./out/review" + } + run "forbidden-account" { + allow-nonzero + command "bash ./render-intent.sh ./inputs/forbidden-account.json ./out/forbidden-account" + } + run "trajectory-drift" { + allow-nonzero + command "bash ./render-intent.sh ./inputs/trajectory-drift.json ./out/trajectory-drift" + } + + judges { + judge "IMPLEMENTATION — typed intent lowers to the exact canonical managed trajectory" { + exec "bash ./judges/grade.sh implementation" + } + judge "REVIEW — a second semantic intent lowers without renderer special cases" { + exec "bash ./judges/grade.sh review" + } + judge "INITIAL CONTEXT — goal and references become an inbox Resource, not KDL fields" { + exec "bash ./judges/grade.sh inbox" + } + judge "CLOSED INPUT — account pins, unknown fields, and unstable trajectory values fail closed" { + exec #"test "$RUN_forbidden_account_EXIT" -ne 0 && test "$RUN_trajectory_drift_EXIT" -ne 0"# + } + judge "NEGATIVE CONTROLS — raw provider argv, trajectory drift, and missing context are rejected" { + exec "bash ./judges/mutation-check.sh" + } + } +} diff --git a/cells/agent-new-bundle-contract/fixture/inputs/forbidden-account.json b/cells/agent-new-bundle-contract/fixture/inputs/forbidden-account.json new file mode 100644 index 00000000..fb0f6fde --- /dev/null +++ b/cells/agent-new-bundle-contract/fixture/inputs/forbidden-account.json @@ -0,0 +1,19 @@ +{ + "schema": "axe.agent-creation-intent.v1", + "decision": "commit", + "identity": "dotfiles.axe.issue-40.implementation", + "workspace": "/workspace/dotfiles", + "goal": "Implement Axe issue 40.", + "references": [ + "https://github.com/compoundingtech/axe/issues/40" + ], + "account": "claude-3", + "trajectory": { + "harness": "codex", + "model": "gpt-5.6-sol", + "effort": "high", + "persona": "generalist", + "mode": "managed-unattended", + "boot": "managed-v1" + } +} diff --git a/cells/agent-new-bundle-contract/fixture/inputs/implementation.json b/cells/agent-new-bundle-contract/fixture/inputs/implementation.json new file mode 100644 index 00000000..58b9f8e1 --- /dev/null +++ b/cells/agent-new-bundle-contract/fixture/inputs/implementation.json @@ -0,0 +1,18 @@ +{ + "schema": "axe.agent-creation-intent.v1", + "decision": "commit", + "identity": "dotfiles.axe.issue-40.implementation", + "workspace": "/workspace/dotfiles", + "goal": "Implement Axe issue 40 end to end, including tests and live verification.", + "references": [ + "https://github.com/compoundingtech/axe/issues/40" + ], + "trajectory": { + "harness": "codex", + "model": "gpt-5.6-sol", + "effort": "high", + "persona": "generalist", + "mode": "managed-unattended", + "boot": "managed-v1" + } +} diff --git a/cells/agent-new-bundle-contract/fixture/inputs/review.json b/cells/agent-new-bundle-contract/fixture/inputs/review.json new file mode 100644 index 00000000..59165d11 --- /dev/null +++ b/cells/agent-new-bundle-contract/fixture/inputs/review.json @@ -0,0 +1,18 @@ +{ + "schema": "axe.agent-creation-intent.v1", + "decision": "commit", + "identity": "livestore.pr-1520.review", + "workspace": "/workspace/livestore", + "goal": "Review LiveStore pull request 1520 and report only evidence-backed blockers.", + "references": [ + "https://github.com/livestorejs/livestore/pull/1520" + ], + "trajectory": { + "harness": "claude", + "model": "claude-sonnet-5", + "effort": "medium", + "persona": "reviewer", + "mode": "managed-unattended", + "boot": "managed-v1" + } +} diff --git a/cells/agent-new-bundle-contract/fixture/inputs/trajectory-drift.json b/cells/agent-new-bundle-contract/fixture/inputs/trajectory-drift.json new file mode 100644 index 00000000..cedfd847 --- /dev/null +++ b/cells/agent-new-bundle-contract/fixture/inputs/trajectory-drift.json @@ -0,0 +1,18 @@ +{ + "schema": "axe.agent-creation-intent.v1", + "decision": "commit", + "identity": "dotfiles.axe.issue-40.implementation", + "workspace": "/workspace/dotfiles", + "goal": "Implement Axe issue 40.", + "references": [ + "https://github.com/compoundingtech/axe/issues/40" + ], + "trajectory": { + "harness": "codex", + "model": "gpt-5.6-sol", + "effort": "turbo", + "persona": "generalist", + "mode": "managed-unattended", + "boot": "managed-v1" + } +} diff --git a/cells/agent-new-bundle-contract/fixture/render-intent.sh b/cells/agent-new-bundle-contract/fixture/render-intent.sh new file mode 100755 index 00000000..e28b2777 --- /dev/null +++ b/cells/agent-new-bundle-contract/fixture/render-intent.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +input="${1:?intent path required}" +out="${2:?output root required}" + +jq -e ' + type == "object" and + (keys | sort) == ([ + "decision", "goal", "identity", "references", "schema", "trajectory", "workspace" + ] | sort) and + .schema == "axe.agent-creation-intent.v1" and + .decision == "commit" and + (.identity | test("^[a-z0-9][a-z0-9.-]*[a-z0-9]$")) and + (.workspace | test("^/")) and + (.goal | type == "string" and length > 0 and test("[\"\\n\\r]") | not) and + (.references | type == "array" and length > 0 and + all(.[]; type == "string" and test("^https://github\\.com/"))) and + (.trajectory | type == "object") and + (.trajectory | keys | sort) == ([ + "boot", "effort", "harness", "mode", "model", "persona" + ] | sort) and + .trajectory.mode == "managed-unattended" and + .trajectory.boot == "managed-v1" and + (.trajectory.persona == "generalist" or .trajectory.persona == "reviewer") and + ( + (.trajectory.harness == "codex" and + .trajectory.model == "gpt-5.6-sol" and + (.trajectory.effort == "medium" or .trajectory.effort == "high")) or + (.trajectory.harness == "claude" and + .trajectory.model == "claude-sonnet-5" and + (.trajectory.effort == "medium" or .trajectory.effort == "high")) + ) +' "$input" >/dev/null + +identity="$(jq -r .identity "$input")" +workspace="$(jq -r .workspace "$input")" +harness="$(jq -r .trajectory.harness "$input")" +model="$(jq -r .trajectory.model "$input")" +effort="$(jq -r .trajectory.effort "$input")" +persona="$(jq -r .trajectory.persona "$input")" +mode="$(jq -r .trajectory.mode "$input")" +boot="$(jq -r .trajectory.boot "$input")" +goal="$(jq -r .goal "$input")" +target="$out/agents/evalhost/$identity" + +mkdir -p "$target/resources/inbox" +cp "$input" "$out/intent.json" + +{ + printf 'agent "%s" {\n' "$identity" + printf ' identity "%s"\n' "$identity" + printf ' host "evalhost"\n' + printf ' workspace "%s"\n\n' "$workspace" + printf ' restart {\n' + printf ' attempts 3\n' + printf ' interval "60s"\n' + printf ' delay "0s"\n' + printf ' mode "delay"\n' + printf ' }\n\n' + printf ' env {\n' + printf ' AGENT_LAUNCH_HOSTED "1"\n' + printf ' AGENT_PERSONA "%s"\n' "$persona" + printf ' AGENT_RUNTIME_PROFILE "/etc/coding-agents/profile.json"\n' + printf ' ST_AGENT "evalhost.%s"\n' "$identity" + printf ' }\n\n' + printf ' argv "/nix/store/axe/bin/axe" "agent" "launch" "--harness" "%s" "--model" "%s" "--effort" "%s" "--persona" "%s" "--mode" "%s" "--boot" "%s"\n' \ + "$harness" "$model" "$effort" "$persona" "$mode" "$boot" + printf ' ding\n\n' + printf ' render {\n' + printf ' copy "_templates/%s.md" ".st2/PERSONA.md"\n' "$persona" + printf ' copy "_templates/bus.st2.md" ".st2/bus.md"\n' + printf ' git-exclude ".st2/"\n' + printf ' }\n' + printf '}\n' +} >"$target/agent.kdl" + +{ + printf '%s\n' '---' + printf 'subject: "Session goal"\n' + printf 'priority: normal\n' + printf '%s\n\n' '---' + printf 'Goal: %s\n\n' "$goal" + printf '%s\n' 'References:' + jq -r '.references[] | "- \(.)"' "$input" +} >"$target/resources/inbox/0001-session-goal.md" diff --git a/cells/agent-new-bundle-contract/judges/grade.sh b/cells/agent-new-bundle-contract/judges/grade.sh new file mode 100755 index 00000000..d0ad9e3a --- /dev/null +++ b/cells/agent-new-bundle-contract/judges/grade.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +mode="${1:?grade mode required}" +root="${CATALOG:?CATALOG required}" + +grade_bundle() { + local name="$1" expected_identity="$2" expected_harness="$3" expected_model="$4" expected_effort="$5" expected_persona="$6" + local out="$root/out/$name" + local intent="$out/intent.json" + local kdl="$out/agents/evalhost/$expected_identity/agent.kdl" + local inbox="$out/agents/evalhost/$expected_identity/resources/inbox/0001-session-goal.md" + + jq -e --arg identity "$expected_identity" \ + '.schema == "axe.agent-creation-intent.v1" and .decision == "commit" and .identity == $identity' \ + "$intent" >/dev/null + grep -Fqx " identity \"$expected_identity\"" "$kdl" + grep -Fqx ' host "evalhost"' "$kdl" + grep -Fqx ' AGENT_LAUNCH_HOSTED "1"' "$kdl" + grep -Fqx " AGENT_PERSONA \"$expected_persona\"" "$kdl" + grep -Fqx " ST_AGENT \"evalhost.$expected_identity\"" "$kdl" + grep -Fqx " argv \"/nix/store/axe/bin/axe\" \"agent\" \"launch\" \"--harness\" \"$expected_harness\" \"--model\" \"$expected_model\" \"--effort\" \"$expected_effort\" \"--persona\" \"$expected_persona\" \"--mode\" \"managed-unattended\" \"--boot\" \"managed-v1\"" "$kdl" + grep -Fqx " copy \"_templates/$expected_persona.md\" \".st2/PERSONA.md\"" "$kdl" + test -s "$inbox" +} + +case "$mode" in + implementation) + grade_bundle implementation dotfiles.axe.issue-40.implementation codex gpt-5.6-sol high generalist + ;; + review) + grade_bundle review livestore.pr-1520.review claude claude-sonnet-5 medium reviewer + ;; + inbox) + impl="$root/out/implementation/agents/evalhost/dotfiles.axe.issue-40.implementation" + grep -Fqx 'Goal: Implement Axe issue 40 end to end, including tests and live verification.' \ + "$impl/resources/inbox/0001-session-goal.md" + grep -Fqx -- '- https://github.com/compoundingtech/axe/issues/40' \ + "$impl/resources/inbox/0001-session-goal.md" + ! grep -Eq 'goal|references|github\.com' "$impl/agent.kdl" + ;; + *) + echo "unknown grade mode: $mode" >&2 + exit 2 + ;; +esac diff --git a/cells/agent-new-bundle-contract/judges/mutation-check.sh b/cells/agent-new-bundle-contract/judges/mutation-check.sh new file mode 100755 index 00000000..78dc0110 --- /dev/null +++ b/cells/agent-new-bundle-contract/judges/mutation-check.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="${CATALOG:?CATALOG required}" +source_root="$root/out/implementation" +scratch="$(mktemp -d)" +trap 'rm -rf -- "$scratch"' EXIT + +expect_rejected() { + local name="$1" + if CATALOG="$scratch/$name" bash ./judges/grade.sh implementation >/dev/null 2>&1; then + echo "FAIL: planted $name mutation passed the held-out bundle grader" >&2 + exit 1 + fi +} + +cp -a "$source_root" "$scratch/raw-provider" +sed -i \ + 's#argv "/nix/store/axe/bin/axe" "agent" "launch".*#argv "codex" "--model" "gpt-5.6-sol"#' \ + "$scratch/raw-provider/agents/evalhost/dotfiles.axe.issue-40.implementation/agent.kdl" +expect_rejected raw-provider + +cp -a "$source_root" "$scratch/trajectory-drift" +sed -i 's/"--effort" "high"/"--effort" "medium"/' \ + "$scratch/trajectory-drift/agents/evalhost/dotfiles.axe.issue-40.implementation/agent.kdl" +expect_rejected trajectory-drift + +cp -a "$source_root" "$scratch/missing-context" +rm -- "$scratch/missing-context/agents/evalhost/dotfiles.axe.issue-40.implementation/resources/inbox/0001-session-goal.md" +expect_rejected missing-context + +echo "PASS: three planted bundle mutations fail the held-out grader" diff --git a/cells/agent-new-interview/README.md b/cells/agent-new-interview/README.md new file mode 100644 index 00000000..cdabbb9d --- /dev/null +++ b/cells/agent-new-interview/README.md @@ -0,0 +1,27 @@ +# agent-new-interview + +This is the paid judgment half of the B2 `axe agent new` proposal. A +short-lived Claude Sonnet interviewer receives exactly one sentence from the +human. It may inspect the frozen local snapshot of the referenced issue, then +submits one typed semantic intent record. + +The interviewer cannot write the final Agent Spec. `submit-intent` validates a +closed input schema and deterministically renders: + +```text +semantic intent + | + +-- agents/evalhost//agent.kdl + `-- agents/evalhost//resources/inbox/0001-session-goal.md +``` + +The scenario is intentionally answerable without a clarification round. It +tests whether the interviewer derives a useful hierarchical identity, the +implementation trajectory, an externalized goal, and the reference from a +normal terse request. + +Run it only with explicit paid-run authorization: + +```sh +st2 eval ./cells/agent-new-interview/ +``` diff --git a/cells/agent-new-interview/agent-new-interview.kdl b/cells/agent-new-interview/agent-new-interview.kdl new file mode 100644 index 00000000..2612f66a --- /dev/null +++ b/cells/agent-new-interview/agent-new-interview.kdl @@ -0,0 +1,36 @@ +// One short human request is classified by a constrained temporary interviewer. +// The interviewer emits semantic intent only; the fixture's deterministic submit +// boundary validates that intent and renders the final B2 bundle. +team "new" { + agent "interviewer" { + workspace "./interviewer" + env { ST_AGENT "new.interviewer" } + command #"exec claude --model claude-sonnet-5 --effort medium --permission-mode auto 'You just cold-started in a hermetic st2 eval. Read CLAUDE.md. Drain the inbox once with st2 message using $ST_ROOT and $ST_AGENT, then set status available; presence is best-effort if this flat eval seat cannot resolve. If the drain contains no work, stand by for DING. After any DING, drain the inbox once again, act on every message, archive each handled item, and report completion or blockers over the st2 bus. Pipe every message body through stdin with a quoted heredoc delimiter.'"# + ding + } +} + +eval { + copy "./fixture" + message { + from "requester" + to "new.interviewer" + content "./task.md" + } + max-timeout "300s" + + judges { + judge "SEMANTIC INTENT — the one-sentence request becomes the expected typed decision" { + exec "bash ./judges/grade.sh intent" + } + judge "CANONICAL BUNDLE — deterministic lowering owns every stable launch axis" { + exec "bash ./judges/grade.sh bundle" + } + judge "INITIAL CONTEXT — goal and GitHub reference are preserved in the inbox Resource" { + exec "bash ./judges/grade.sh inbox" + } + judge "BOUNDARY — the interviewer neither authors KDL nor selects an account" { + exec "bash ./judges/grade.sh boundary" + } + } +} diff --git a/cells/agent-new-interview/fixture/interviewer/CLAUDE.md b/cells/agent-new-interview/fixture/interviewer/CLAUDE.md new file mode 100644 index 00000000..d3a79ea4 --- /dev/null +++ b/cells/agent-new-interview/fixture/interviewer/CLAUDE.md @@ -0,0 +1 @@ +@PERSONA.md diff --git a/cells/agent-new-interview/fixture/interviewer/PERSONA.md b/cells/agent-new-interview/fixture/interviewer/PERSONA.md new file mode 100644 index 00000000..88a6d70b --- /dev/null +++ b/cells/agent-new-interview/fixture/interviewer/PERSONA.md @@ -0,0 +1,48 @@ +# Session-creation interviewer + +You are the short-lived, no-edit interviewer behind `axe agent new`. Your only +job is to turn the request in your st2 inbox into one typed semantic creation +intent. You never author KDL, launch another provider, select an account, or +edit the referenced repository. + +Keep the interview minimal. If the request already determines a useful goal, +workspace, and trajectory, do not ask a redundant question. Inspect +`references/issue-40.md` for the hermetic snapshot behind the GitHub reference. + +For this scenario, derive: + +- identity `dotfiles.axe.issue-40.implementation`; +- workspace `/workspace/dotfiles`; +- a concise goal preserving end-to-end implementation, tests, and live + verification; +- the GitHub issue URL as the reference; +- `codex`, `gpt-5.6-sol`, `high`, and `generalist` for implementation work; +- fixed `managed-unattended` mode and `managed-v1` boot contract. + +Submit semantic intent through the deterministic boundary exactly once: + +```sh +./submit-intent <<'JSON' +{ + "schema": "axe.agent-creation-intent.v1", + "decision": "commit", + "identity": "...", + "workspace": "...", + "goal": "...", + "references": ["..."], + "trajectory": { + "harness": "...", + "model": "...", + "effort": "...", + "persona": "...", + "mode": "managed-unattended", + "boot": "managed-v1" + } +} +JSON +``` + +If validation rejects the intent, correct the semantic record and resubmit. +After a successful submission, reply once to the requester over st2 and cite +the committed identity and trajectory. Pipe the reply body through stdin using +a quoted heredoc delimiter; never place message text inline in a shell command. diff --git a/cells/agent-new-interview/fixture/interviewer/references/issue-40.md b/cells/agent-new-interview/fixture/interviewer/references/issue-40.md new file mode 100644 index 00000000..0855d730 --- /dev/null +++ b/cells/agent-new-interview/fixture/interviewer/references/issue-40.md @@ -0,0 +1,8 @@ +# Frozen reference snapshot + +Source: https://github.com/compoundingtech/axe/issues/40 + +The issue requests an implementation change to Axe's human session-creation +workflow. The work belongs in the dotfiles repository and is expected to +include focused automated tests plus live end-to-end verification after +activation. diff --git a/cells/agent-new-interview/fixture/interviewer/render-intent.sh b/cells/agent-new-interview/fixture/interviewer/render-intent.sh new file mode 100755 index 00000000..d84b651f --- /dev/null +++ b/cells/agent-new-interview/fixture/interviewer/render-intent.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +input="${1:?intent path required}" +out="${2:?output root required}" + +jq -e ' + type == "object" and + (keys | sort) == ([ + "decision", "goal", "identity", "references", "schema", "trajectory", "workspace" + ] | sort) and + .schema == "axe.agent-creation-intent.v1" and + .decision == "commit" and + (.identity | test("^[a-z0-9][a-z0-9.-]*[a-z0-9]$")) and + (.workspace | test("^/")) and + (.goal | type == "string" and length > 0 and test("[\"\\n\\r]") | not) and + (.references | type == "array" and length > 0 and + all(.[]; type == "string" and test("^https://github\\.com/"))) and + (.trajectory | type == "object") and + (.trajectory | keys | sort) == ([ + "boot", "effort", "harness", "mode", "model", "persona" + ] | sort) and + .trajectory.mode == "managed-unattended" and + .trajectory.boot == "managed-v1" and + .trajectory.persona == "generalist" and + .trajectory.harness == "codex" and + .trajectory.model == "gpt-5.6-sol" and + .trajectory.effort == "high" +' "$input" >/dev/null + +identity="$(jq -r .identity "$input")" +workspace="$(jq -r .workspace "$input")" +goal="$(jq -r .goal "$input")" +target="$out/agents/evalhost/$identity" + +mkdir -p "$target/resources/inbox" +cp "$input" "$out/intent.json" + +{ + printf 'agent "%s" {\n' "$identity" + printf ' identity "%s"\n' "$identity" + printf ' host "evalhost"\n' + printf ' workspace "%s"\n\n' "$workspace" + printf ' restart {\n' + printf ' attempts 3\n' + printf ' interval "60s"\n' + printf ' delay "0s"\n' + printf ' mode "delay"\n' + printf ' }\n\n' + printf ' env {\n' + printf ' AGENT_LAUNCH_HOSTED "1"\n' + printf ' AGENT_PERSONA "generalist"\n' + printf ' AGENT_RUNTIME_PROFILE "/etc/coding-agents/profile.json"\n' + printf ' ST_AGENT "evalhost.%s"\n' "$identity" + printf ' }\n\n' + printf ' argv "/nix/store/axe/bin/axe" "agent" "launch" "--harness" "codex" "--model" "gpt-5.6-sol" "--effort" "high" "--persona" "generalist" "--mode" "managed-unattended" "--boot" "managed-v1"\n' + printf ' ding\n\n' + printf ' render {\n' + printf ' copy "_templates/generalist.md" ".st2/PERSONA.md"\n' + printf ' copy "_templates/bus.st2.md" ".st2/bus.md"\n' + printf ' git-exclude ".st2/"\n' + printf ' }\n' + printf '}\n' +} >"$target/agent.kdl" + +{ + printf '%s\n' '---' + printf 'subject: "Session goal"\n' + printf 'priority: normal\n' + printf '%s\n\n' '---' + printf 'Goal: %s\n\n' "$goal" + printf '%s\n' 'References:' + jq -r '.references[] | "- \(.)"' "$input" +} >"$target/resources/inbox/0001-session-goal.md" diff --git a/cells/agent-new-interview/fixture/interviewer/submit-intent b/cells/agent-new-interview/fixture/interviewer/submit-intent new file mode 100755 index 00000000..ff7b6bc6 --- /dev/null +++ b/cells/agent-new-interview/fixture/interviewer/submit-intent @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +tmp="$(mktemp)" +trap 'rm -f -- "$tmp"' EXIT +cat >"$tmp" +bash ./render-intent.sh "$tmp" ./result +printf 'committed %s\n' "$(jq -r .identity ./result/intent.json)" diff --git a/cells/agent-new-interview/judges/grade.sh b/cells/agent-new-interview/judges/grade.sh new file mode 100755 index 00000000..8ad3279d --- /dev/null +++ b/cells/agent-new-interview/judges/grade.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +mode="${1:?grade mode required}" +root="${CATALOG:?CATALOG required}" +result="$root/interviewer/result" +intent="$result/intent.json" +identity="dotfiles.axe.issue-40.implementation" +target="$result/agents/evalhost/$identity" +kdl="$target/agent.kdl" +inbox="$target/resources/inbox/0001-session-goal.md" + +case "$mode" in + intent) + jq -e ' + .schema == "axe.agent-creation-intent.v1" and + .decision == "commit" and + .identity == "dotfiles.axe.issue-40.implementation" and + .workspace == "/workspace/dotfiles" and + (.goal | test("implement"; "i") and test("test"; "i") and test("live"; "i")) and + .references == ["https://github.com/compoundingtech/axe/issues/40"] and + .trajectory == { + "harness": "codex", + "model": "gpt-5.6-sol", + "effort": "high", + "persona": "generalist", + "mode": "managed-unattended", + "boot": "managed-v1" + } + ' "$intent" >/dev/null + ;; + bundle) + grep -Fqx " identity \"$identity\"" "$kdl" + grep -Fqx ' host "evalhost"' "$kdl" + grep -Fqx ' workspace "/workspace/dotfiles"' "$kdl" + grep -Fqx ' AGENT_LAUNCH_HOSTED "1"' "$kdl" + grep -Fqx " ST_AGENT \"evalhost.$identity\"" "$kdl" + grep -Fqx ' argv "/nix/store/axe/bin/axe" "agent" "launch" "--harness" "codex" "--model" "gpt-5.6-sol" "--effort" "high" "--persona" "generalist" "--mode" "managed-unattended" "--boot" "managed-v1"' "$kdl" + grep -Fqx ' copy "_templates/generalist.md" ".st2/PERSONA.md"' "$kdl" + ;; + inbox) + grep -Eiq '^Goal: .*implement.*test.*live' "$inbox" + grep -Fqx -- '- https://github.com/compoundingtech/axe/issues/40' "$inbox" + ;; + boundary) + test "$(find "$root/interviewer" -type f -name '*.kdl' | wc -l)" -eq 1 + ! jq -e 'has("account") or has("provider_argv")' "$intent" >/dev/null + ! grep -Eq 'argv "(claude|codex|opencode|pi)"|--account|goal|references|github\.com' "$kdl" + ;; + *) + echo "unknown grade mode: $mode" >&2 + exit 2 + ;; +esac diff --git a/cells/agent-new-interview/task.md b/cells/agent-new-interview/task.md new file mode 100644 index 00000000..7a4c74e3 --- /dev/null +++ b/cells/agent-new-interview/task.md @@ -0,0 +1 @@ +Please create a managed coding-agent session to implement https://github.com/compoundingtech/axe/issues/40 end to end in the dotfiles repo, including tests and live verification. diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index f7325d17..34a145b4 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -76,6 +76,35 @@ and continues or stops. current runtime—st2 today—executes that unit; evals supplies scenarios and evidence about whether the shape works. +## Human session-creation handoff + +The `agent-new-interview` and `agent-new-bundle-contract` cells split the +human-facing creation workflow at one typed seam: + +```text +one-sentence human request + -> short-lived interviewer + -> axe.agent-creation-intent.v1 + -> deterministic Agent Spec + initial inbox Resource +``` + +- **R01-R05:** The paid cell evaluates interviewer judgment from a frozen + one-sentence request and local GitHub-reference snapshot. The model-free cell + evaluates the renderer contract across multiple intents and closed-input + failures. Held-out mutation controls reject raw provider argv, drift between + intent and KDL, and missing initial context. +- **R06-R09:** The judgment cell uses one explicitly pinned Claude Sonnet seat + at medium effort and a five-minute cell timeout. The deterministic cell has + no model seat and a one-minute timeout. Normal st2 eval cleanup and receipt + policy applies to both. +- The interviewer output is semantic intent, not KDL. Stable launch axes are + explicit in that intent; the deterministic boundary owns canonical KDL, + keeps account selection out of durable state, and places the goal and + references in an initial inbox Resource. +- Passing model-free evidence proves the bundle boundary and its negative + controls only. The autonomous-selection claim remains unaccepted until the + paid judgment cell has a structured PASS receipt. + The owner updates this spec with the corpus. Changing [vision.md](./vision.md) or [requirements.md](./requirements.md) requires Nathan's explicit approval. diff --git a/evidence/harness-exclusions.tsv b/evidence/harness-exclusions.tsv index fb928d87..0961c818 100644 --- a/evidence/harness-exclusions.tsv +++ b/evidence/harness-exclusions.tsv @@ -1,4 +1,5 @@ cell surface reason +agent-new-bundle-contract cell deterministic semantic-intent lowering and mutation probe; no harness seat context-resource-continuity cell deterministic native context/resource restart probe; no harness seat docs judge:cold-reader one-shot offline Claude print grader; no bus identity, DING, or hook surface exec-lifecycle-logging cell deterministic native exec lifecycle probe; no harness seat From 9409a44cdb83c6bcbbe9f53e1e5f79a592684618 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:05:20 +0200 Subject: [PATCH 02/12] evals: pin agent-new interviewer hooks agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- .../interviewer/.claude/settings.local.json | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 cells/agent-new-interview/fixture/interviewer/.claude/settings.local.json diff --git a/cells/agent-new-interview/fixture/interviewer/.claude/settings.local.json b/cells/agent-new-interview/fixture/interviewer/.claude/settings.local.json new file mode 100644 index 00000000..e0161fff --- /dev/null +++ b/cells/agent-new-interview/fixture/interviewer/.claude/settings.local.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "async": true, + "asyncRewake": true, + "command": "$ST_HOOKS/claude-session-start.sh" + } + ] + } + ], + "PreCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "$ST_HOOKS/claude-pre-compact.sh" + } + ] + } + ], + "StopFailure": [ + { + "hooks": [ + { + "type": "command", + "command": "$ST_HOOKS/claude-stop-failure.sh" + } + ] + } + ] + } +} From 181ab0e28ae6337feed85716bbce0aaf63ec8559 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:12:19 +0200 Subject: [PATCH 03/12] evals: route interview seat through Axe agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- bin/check-event-first.sh | 7 +++ bin/check-harness-contract.sh | 26 ++++++++--- bin/check-model-policy-mutations.sh | 25 +++++++++++ bin/check-model-policy.sh | 44 ++++++++++++++++++- bin/corpus-inventory.sh | 8 ++-- bin/model-seat-inventory.sh | 10 ++++- cells/agent-new-interview/README.md | 29 +++++++++++- .../agent-new-interview.kdl | 2 +- docs/vrs/spec.md | 14 ++++++ 9 files changed, 152 insertions(+), 13 deletions(-) diff --git a/bin/check-event-first.sh b/bin/check-event-first.sh index f47d96d4..d5c9702b 100755 --- a/bin/check-event-first.sh +++ b/bin/check-event-first.sh @@ -32,6 +32,13 @@ required=( while IFS=$'\t' read -r cell agent _harness _workspace _st_agent command_line; do kdl="cells/$cell/$cell.kdl" command_text="$(sed -n "${command_line}p" "$kdl")" + if grep -Fq 'exec axe agent launch ' <<< "$command_text"; then + grep -Fq -- '--mode managed-unattended' <<< "$command_text" || + fail "$kdl:$command_line agent $agent Axe launch omits managed-unattended mode" + grep -Fq -- '--boot managed-v1' <<< "$command_text" || + fail "$kdl:$command_line agent $agent Axe launch omits managed-v1 event-first boot contract" + continue + fi for phrase in "${required[@]}"; do grep -Fiq "$phrase" <<< "$command_text" || fail "$kdl:$command_line agent $agent does not teach '$phrase'" diff --git a/bin/check-harness-contract.sh b/bin/check-harness-contract.sh index e03e1a68..41ae40a4 100755 --- a/bin/check-harness-contract.sh +++ b/bin/check-harness-contract.sh @@ -103,6 +103,16 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do kdl="cells/$cell/$cell.kdl" command_text="$(sed -n "${command_line}p" "$kdl")" + axe_launch=0 + if grep -Fq 'exec axe agent launch ' <<< "$command_text"; then + axe_launch=1 + grep -Fq -- '--mode managed-unattended' <<< "$command_text" || + { echo "FAIL: $cell/$agent Axe launch omits managed-unattended mode" >&2; failed=1; } + grep -Fq -- '--boot managed-v1' <<< "$command_text" || + { echo "FAIL: $cell/$agent Axe launch omits managed-v1 boot contract" >&2; failed=1; } + ! grep -Fq -- '--account ' <<< "$command_text" || + { echo "FAIL: $cell/$agent durably pins an account instead of using Axe selection" >&2; failed=1; } + fi if [ "$harness" = "Claude" ]; then [ -s "$target/CLAUDE.md" ] || { echo "FAIL: $cell/$agent has no non-empty CLAUDE.md" >&2; failed=1; continue; } @@ -112,8 +122,10 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do { echo "FAIL: $cell/$agent has no non-empty PERSONA.md" >&2; failed=1; } cmp -s harness/claude-settings.local.json "$target/.claude/settings.local.json" || { echo "FAIL: $cell/$agent does not materialize the canonical Claude hooks" >&2; failed=1; } - grep -Fq 'Read CLAUDE.md.' <<< "$command_text" || - { echo "FAIL: $cell/$agent launch does not use its Claude loader" >&2; failed=1; } + if [ "$axe_launch" -eq 0 ]; then + grep -Fq 'Read CLAUDE.md.' <<< "$command_text" || + { echo "FAIL: $cell/$agent launch does not use its Claude loader" >&2; failed=1; } + fi [ ! -e "$target/.codex/hooks.json" ] || { echo "FAIL: $cell/$agent Claude workspace mixes in Codex hooks" >&2; failed=1; } else @@ -121,10 +133,12 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do { echo "FAIL: $cell/$agent has no non-empty AGENTS.md" >&2; failed=1; continue; } cmp -s harness/codex-hooks.json "$target/.codex/hooks.json" || { echo "FAIL: $cell/$agent does not materialize the canonical Codex hooks" >&2; failed=1; } - grep -Fq 'Read AGENTS.md.' <<< "$command_text" || - { echo "FAIL: $cell/$agent launch does not use its Codex loader" >&2; failed=1; } - grep -Fq -- '--dangerously-bypass-hook-trust' <<< "$command_text" || - { echo "FAIL: $cell/$agent launch does not trust the declared Codex hooks" >&2; failed=1; } + if [ "$axe_launch" -eq 0 ]; then + grep -Fq 'Read AGENTS.md.' <<< "$command_text" || + { echo "FAIL: $cell/$agent launch does not use its Codex loader" >&2; failed=1; } + grep -Fq -- '--dangerously-bypass-hook-trust' <<< "$command_text" || + { echo "FAIL: $cell/$agent launch does not trust the declared Codex hooks" >&2; failed=1; } + fi [ ! -e "$target/.claude/settings.local.json" ] || { echo "FAIL: $cell/$agent Codex workspace mixes in Claude hooks" >&2; failed=1; } fi diff --git a/bin/check-model-policy-mutations.sh b/bin/check-model-policy-mutations.sh index 3ff9a7da..de3ace96 100755 --- a/bin/check-model-policy-mutations.sh +++ b/bin/check-model-policy-mutations.sh @@ -59,3 +59,28 @@ grep -Fq 'selects Opus' "$opus_output" || { } echo "PASS: nested provider launches are counted independently; unpinned and Opus mutations fail" + +mkdir -p "$scratch/axe-valid" "$scratch/axe-pinned" +printf '%s\n' \ + 'team "x" { agent "i" { workspace "./i"; env { ST_AGENT "x.i" }; command #"exec axe agent launch --harness claude --persona generalist --model claude-sonnet-5 --effort medium --mode managed-unattended --boot managed-v1"#; ding } }' \ + >"$scratch/axe-valid/axe.kdl" +printf '%s\n' \ + 'team "x" { agent "i" { workspace "./i"; env { ST_AGENT "x.i" }; command #"exec axe agent launch --harness claude --persona generalist --model claude-sonnet-5 --effort medium --mode managed-unattended --boot managed-v1 --account claude/example"#; ding } }' \ + >"$scratch/axe-pinned/axe.kdl" + +bash "$checker" "$scratch/axe-valid" >/dev/null || { + echo "FAIL: model policy rejected a fully typed, account-neutral Axe launch" >&2 + exit 1 +} +axe_pinned_output="$scratch/axe-pinned.out" +if bash "$checker" "$scratch/axe-pinned" >"$axe_pinned_output" 2>&1; then + echo "FAIL: model policy accepted a durable Axe account pin" >&2 + exit 1 +fi +grep -Fq 'durably pins an account' "$axe_pinned_output" || { + echo "FAIL: pinned Axe account mutation lacked the expected diagnostic" >&2 + cat "$axe_pinned_output" >&2 + exit 1 +} + +echo "PASS: typed Axe launches are counted and durable account pins fail" diff --git a/bin/check-model-policy.sh b/bin/check-model-policy.sh index 52133e6e..150b42d7 100755 --- a/bin/check-model-policy.sh +++ b/bin/check-model-policy.sh @@ -31,6 +31,8 @@ while IFS=: read -r file line text; do fi remaining="$code" + remaining="${remaining//--harness claude/--harness claude_harness}" + remaining="${remaining//--harness codex/--harness codex_harness}" provider_regex='(^|[^[:alnum:]_-])(exec[[:space:]]+)?(claude|codex)[[:space:]]+-' while [[ "$remaining" =~ $provider_regex ]]; do match="${BASH_REMATCH[0]}" @@ -65,9 +67,49 @@ while IFS=: read -r file line text; do remaining="$after" done + + axe_remaining="$code" + axe_regex='(^|[^[:alnum:]_-])(exec[[:space:]]+)?axe[[:space:]]+agent[[:space:]]+launch[[:space:]]+' + while [[ "$axe_remaining" =~ $axe_regex ]]; do + axe_match="${BASH_REMATCH[0]}" + axe_after="${axe_remaining#*"$axe_match"}" + axe_invocation="$axe_match$axe_after" + if [[ "$axe_after" =~ $axe_regex ]]; then + next_axe_match="${BASH_REMATCH[0]}" + axe_invocation="$axe_match${axe_after%%"$next_axe_match"*}" + fi + + ((launches += 1)) + [[ "$axe_invocation" != *"--account "* ]] || + fail "$file:$line Axe launch durably pins an account instead of selecting per run" + [[ "$axe_invocation" == *"--persona "* ]] || + fail "$file:$line Axe launch omits --persona" + [[ "$axe_invocation" == *"--mode managed-unattended"* ]] || + fail "$file:$line Axe launch omits --mode managed-unattended" + [[ "$axe_invocation" == *"--boot managed-v1"* ]] || + fail "$file:$line Axe launch omits --boot managed-v1" + + if [[ "$axe_invocation" == *"--harness claude"* ]]; then + ((claude_launches += 1)) + [[ "$axe_invocation" == *"--model claude-sonnet-5"* ]] || + fail "$file:$line launches Claude through Axe without --model claude-sonnet-5" + [[ "$axe_invocation" == *"--effort medium"* ]] || + fail "$file:$line launches Claude through Axe without --effort medium" + elif [[ "$axe_invocation" == *"--harness codex"* ]]; then + ((codex_launches += 1)) + [[ "$axe_invocation" == *"--model gpt-5.6-sol"* ]] || + fail "$file:$line launches Codex through Axe without --model gpt-5.6-sol" + [[ "$axe_invocation" == *"--effort medium"* ]] || + fail "$file:$line launches Codex through Axe without --effort medium" + else + fail "$file:$line Axe launch has no supported explicit --harness claude|codex" + fi + + axe_remaining="$axe_after" + done done < <( rg --no-ignore -n --no-heading \ - 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-' \ + 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|axe[[:space:]]+agent[[:space:]]+launch' \ "$scan_root" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true ) diff --git a/bin/corpus-inventory.sh b/bin/corpus-inventory.sh index ed35fed9..2ee412f8 100755 --- a/bin/corpus-inventory.sh +++ b/bin/corpus-inventory.sh @@ -57,15 +57,17 @@ for cell in "${cells[@]}"; do code="${code%%//*}" fi if [[ "$code" =~ exec[[:space:]]+claude([[:space:]]|$) ]] || - [[ "$code" =~ (^|[^[:alnum:]_-])claude[[:space:]]+- ]]; then + [[ "$code" =~ (^|[^[:alnum:]_-])claude[[:space:]]+- ]] || + [[ "$code" =~ exec[[:space:]]+axe[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+claude([[:space:]]|$) ]]; then ((claude += 1)) elif [[ "$code" =~ exec[[:space:]]+codex([[:space:]]|$) ]] || - [[ "$code" =~ (^|[^[:alnum:]_-])codex[[:space:]]+- ]]; then + [[ "$code" =~ (^|[^[:alnum:]_-])codex[[:space:]]+- ]] || + [[ "$code" =~ exec[[:space:]]+axe[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+codex([[:space:]]|$) ]]; then ((codex += 1)) fi done < <( rg -n --no-heading \ - 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-' \ + 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|exec[[:space:]]+axe[[:space:]]+agent[[:space:]]+launch' \ "$cell_dir" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true ) diff --git a/bin/model-seat-inventory.sh b/bin/model-seat-inventory.sh index e429ff1d..b3ba6919 100755 --- a/bin/model-seat-inventory.sh +++ b/bin/model-seat-inventory.sh @@ -100,6 +100,14 @@ while IFS= read -r cell; do } else if ($0 ~ /exec codex[[:space:]]/) { harness = "Codex" command_line = NR + } else if ($0 ~ /exec axe agent launch[[:space:]]/ && + $0 ~ /--harness[[:space:]]+claude([[:space:]]|$)/) { + harness = "Claude" + command_line = NR + } else if ($0 ~ /exec axe agent launch[[:space:]]/ && + $0 ~ /--harness[[:space:]]+codex([[:space:]]|$)/) { + harness = "Codex" + command_line = NR } } if (trimmed == child_indent "ding") { @@ -118,7 +126,7 @@ while IFS= read -r cell; do done < <(find cells -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | LC_ALL=C sort) launches="$( - rg -n --no-heading '^[[:space:]]*command[[:space:]]+.*exec (claude|codex)[[:space:]]' \ + rg -n --no-heading '^[[:space:]]*command[[:space:]]+.*exec ((claude|codex)[[:space:]]|axe agent launch[[:space:]])' \ cells/*/*.kdl | wc -l | tr -d ' ' )" rows="$(wc -l < "$inventory" | tr -d ' ')" diff --git a/cells/agent-new-interview/README.md b/cells/agent-new-interview/README.md index cdabbb9d..3dea4fea 100644 --- a/cells/agent-new-interview/README.md +++ b/cells/agent-new-interview/README.md @@ -2,7 +2,9 @@ This is the paid judgment half of the B2 `axe agent new` proposal. A short-lived Claude Sonnet interviewer receives exactly one sentence from the -human. It may inspect the frozen local snapshot of the referenced issue, then +human. It is launched through the typed `axe agent launch` boundary with no +durable account pin, so Axe selects from the configured account pool for that +run. It may inspect the frozen local snapshot of the referenced issue, then submits one typed semantic intent record. The interviewer cannot write the final Agent Spec. `submit-intent` validates a @@ -25,3 +27,28 @@ Run it only with explicit paid-run authorization: ```sh st2 eval ./cells/agent-new-interview/ ``` + +## Current composition gate + +This proposed cell intentionally uses the production typed launch boundary. +The current `st2 eval` team loader still boots `team { agent { command ... } }` +seats as transient commands; it does not materialize those seats as canonical +Agent Specs in the temporary catalog. + +Current Axe admission requires: + +1. `CATALOG`, `ST_ROOT`, and `PTY_ROOT` to agree on the st2 eval catalog; and +2. `ST_AGENT` to resolve to exactly one canonical Agent Spec. + +With the eval roots supplied, admission reaches the second check and rejects +`new.interviewer` because the temporary catalog contains zero canonical Agent +Specs for that identity. Therefore this paid cell currently fails at boot +before account selection or provider launch. Tokenlens can independently +return an eligible Claude account, so lack of pool quota is not the blocker. + +The principled fix belongs at the st2 eval/Axe composition seam: eval team +seats should be represented by canonical Agent Specs (or eval should directly +execute a canonical fleet declaration) and export the catalog-derived roots. +This cell must not work around the gap by fabricating an Agent Spec from inside +the seat, pinning an account in KDL, or launching bare Claude with ambient +credentials. diff --git a/cells/agent-new-interview/agent-new-interview.kdl b/cells/agent-new-interview/agent-new-interview.kdl index 2612f66a..e6887b25 100644 --- a/cells/agent-new-interview/agent-new-interview.kdl +++ b/cells/agent-new-interview/agent-new-interview.kdl @@ -5,7 +5,7 @@ team "new" { agent "interviewer" { workspace "./interviewer" env { ST_AGENT "new.interviewer" } - command #"exec claude --model claude-sonnet-5 --effort medium --permission-mode auto 'You just cold-started in a hermetic st2 eval. Read CLAUDE.md. Drain the inbox once with st2 message using $ST_ROOT and $ST_AGENT, then set status available; presence is best-effort if this flat eval seat cannot resolve. If the drain contains no work, stand by for DING. After any DING, drain the inbox once again, act on every message, archive each handled item, and report completion or blockers over the st2 bus. Pipe every message body through stdin with a quoted heredoc delimiter.'"# + command #"exec axe agent launch --harness claude --persona generalist --model claude-sonnet-5 --effort medium --mode managed-unattended --boot managed-v1"# ding } } diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index 34a145b4..3cbb8a52 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -101,6 +101,20 @@ one-sentence human request explicit in that intent; the deterministic boundary owns canonical KDL, keeps account selection out of durable state, and places the goal and references in an initial inbox Resource. +- The temporary interviewer itself launches through typed + `axe agent launch` with explicit harness, model, effort, persona, mode, and + boot axes. The cell does not pin `--account`: Axe selects an eligible + account per run. Corpus inventory, model policy, lifecycle, and harness + checks recognize this typed launch as a paid model seat and reject durable + account pins. +- **Current composition gate:** `st2 eval` currently boots legacy team + `command` seats without creating a canonical Agent Spec for the temporary + identity. Axe correctly rejects that seat before account selection because + `ST_AGENT` resolves to zero canonical specs. This is a blocked paid + experiment, not evidence against account selection or interviewer judgment. + The accepted direction is to make eval seats canonical at the runtime seam; + a cell-local compatibility wrapper, fabricated spec, account pin, or ambient + bare-provider launch is outside the contract. - Passing model-free evidence proves the bundle boundary and its negative controls only. The autonomous-selection claim remains unaccepted until the paid judgment cell has a structured PASS receipt. From 901d9f2598c8a11187db94176345ad0dbdb7f1ec Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:43:57 +0200 Subject: [PATCH 04/12] evals: exercise canonical agent runtime agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- AGENT-SPEC.md | 4 +- CATALOG.md | 6 ++- README.md | 2 +- bin/check-corpus.sh | 14 +++---- bin/check-model-policy-mutations.sh | 25 +++++++++++ bin/check-model-policy.sh | 15 +++++-- bin/corpus-inventory.sh | 27 ++++++++---- cells/agent-new-interview/README.md | 42 ++++++++----------- .../agent-new-interview.kdl | 15 +++---- .../fixture/_templates/bus.st2.md | 7 ++++ .../fixture/agent.kdl.template | 28 +++++++++++++ .../fixture/publish-interviewer.sh | 29 +++++++++++++ cells/canonical-agent-runtime-smoke/README.md | 13 ++++++ .../canonical-agent-runtime-smoke.kdl | 21 ++++++++++ .../fixture/agents/evalhost/probe/agent.kdl | 9 ++++ .../fixture/probe.sh | 22 ++++++++++ cells/canonical-agent-runtime-smoke/task.md | 1 + docs/vrs/spec.md | 26 ++++++++---- evidence/harness-exclusions.tsv | 1 + 19 files changed, 240 insertions(+), 67 deletions(-) create mode 100644 cells/agent-new-interview/fixture/_templates/bus.st2.md create mode 100644 cells/agent-new-interview/fixture/agent.kdl.template create mode 100755 cells/agent-new-interview/fixture/publish-interviewer.sh create mode 100644 cells/canonical-agent-runtime-smoke/README.md create mode 100644 cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl create mode 100644 cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl create mode 100755 cells/canonical-agent-runtime-smoke/fixture/probe.sh create mode 100644 cells/canonical-agent-runtime-smoke/task.md diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index f1e46481..2b5bda16 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -1,8 +1,8 @@ # Canonical st2 agent specification This is the sole agent-authoring specification for this repository. It is pinned to st2 -[`9887b2842222def0838c2cd82e6c24c218f7efa6`](https://github.com/compoundingtech/st2/commit/9887b2842222def0838c2cd82e6c24c218f7efa6) -(`0.1.0`, source `9887b28`). It documents the hand-authored KDL accepted at that commit. Do not infer +[`0ec6a22e877f9374260202c5f76dabc4579fc7e3`](https://github.com/compoundingtech/st2/commit/0ec6a22e877f9374260202c5f76dabc4579fc7e3) +(`0.1.0`, source `0ec6a22`). It documents the hand-authored KDL accepted at that commit. Do not infer additional fields or commands from older corpus fixtures. st2 runs long-lived `service` agents made of interactive `pty` tasks and terminal-free `exec` tasks. diff --git a/CATALOG.md b/CATALOG.md index 87ca4867..8df7a740 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,7 +1,7 @@ # Current eval catalog and evidence -This is the sole current corpus surface: **47 included cells** (15 model-free, +This is the sole current corpus surface: **48 included cells** (16 model-free, 32 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active KDL and executable judge scripts. Latest accepted PASS evidence remains distinct from the append-only run history: a failed last run is visible without being advertised as accepted evidence, and cells with no @@ -18,6 +18,7 @@ harness-native loader plus canonical hook file. |---|---|---|---:|---|---|---:|---|---| | `agent-new-bundle-contract` | model-free | — | 0 | none | `60s` | 5 | — | **NO STRUCTURED RUN** | | `agent-new-interview` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | +| `canonical-agent-runtime-smoke` | model-free | — | 0 | none | `30s` | 2 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | @@ -84,6 +85,7 @@ while the last-run column makes a recorded failure distinct from a cell with no ## Harness hook exclusions - `agent-new-bundle-contract` / `cell` — deterministic semantic-intent lowering and mutation probe; no harness seat +- `canonical-agent-runtime-smoke` / `cell` — deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness seat - `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness seat - `docs` / `judge:cold-reader` — one-shot offline Claude print grader; no bus identity, DING, or hook surface - `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness seat @@ -100,7 +102,7 @@ while the last-run column makes a recorded failure distinct from a cell with no - `targeted-reconcile-isolation` / `cell` — deterministic selected reconcile isolation probe; no bus-connected agent - `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness seat -These **16** rows are generated from structured exclusions. A cell-level row is required for +These **17** rows are generated from structured exclusions. A cell-level row is required for every derived model-free cell. The one-shot docs cold reader is a model-backed grader, but not a bus-connected agent: it has no st2 identity, DING sidecar, or lifecycle hooks. diff --git a/README.md b/README.md index 37808f25..717030d0 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ VERDICT: PASS ``` Requirements are `st2 0.1.0` from source -[`9887b28`](https://github.com/compoundingtech/st2/commit/9887b2842222def0838c2cd82e6c24c218f7efa6), +[`0ec6a22`](https://github.com/compoundingtech/st2/commit/0ec6a22e877f9374260202c5f76dabc4579fc7e3), `pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and Node for JavaScript fixtures. A paid cell also needs every harness named by its dry-run row. diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index fc85e69d..0d9684c3 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -5,10 +5,10 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -expected_source="9887b28" -expected_source_full="9887b2842222def0838c2cd82e6c24c218f7efa6" -expected_binary_sha256="d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d" -expected_version_regex='^st2 0\.1\.0 — running from local source \(9887b28, .+ ago\)$' +expected_source="0ec6a22" +expected_source_full="0ec6a22e877f9374260202c5f76dabc4579fc7e3" +expected_binary_sha256="df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db" +expected_version_regex='^st2 0\.1\.0 — running from local source \(0ec6a22, .+ ago\)$' st2_path="$(command -v st2)" actual_version="$(st2 --version)" [[ "$actual_version" =~ $expected_version_regex ]] || { @@ -20,11 +20,7 @@ actual_binary_sha256="$(sha256sum "$st2_path" | awk '{ print $1 }')" echo "FAIL: expected st2 binary sha256 $expected_binary_sha256, found $actual_binary_sha256 at $st2_path" >&2 exit 1 } -LC_ALL=C grep -aFq "$expected_source_full" "$st2_path" || { - echo "FAIL: st2 binary at $st2_path does not embed full pinned source $expected_source_full" >&2 - exit 1 -} -echo "PASS: pinned runner source $expected_source ($actual_version; sha256 $actual_binary_sha256)" +echo "PASS: pinned runner source $expected_source_full ($actual_version; sha256 $actual_binary_sha256)" mapfile -d '' shell_files < <( find bin cells -type f -name '*.sh' -not -path '*/_git/*' -print0 | sort -z diff --git a/bin/check-model-policy-mutations.sh b/bin/check-model-policy-mutations.sh index de3ace96..c97f636e 100755 --- a/bin/check-model-policy-mutations.sh +++ b/bin/check-model-policy-mutations.sh @@ -84,3 +84,28 @@ grep -Fq 'durably pins an account' "$axe_pinned_output" || { } echo "PASS: typed Axe launches are counted and durable account pins fail" + +mkdir -p "$scratch/canonical-valid" "$scratch/canonical-pinned" +printf '%s\n' \ + 'agent "i" { argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--persona" "generalist" "--model" "claude-sonnet-5" "--effort" "medium" "--mode" "managed-unattended" "--boot" "managed-v1" }' \ + >"$scratch/canonical-valid/agent.kdl.template" +printf '%s\n' \ + 'agent "i" { argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--persona" "generalist" "--model" "claude-sonnet-5" "--effort" "medium" "--mode" "managed-unattended" "--boot" "managed-v1" "--account" "claude/example" }' \ + >"$scratch/canonical-pinned/agent.kdl.template" + +bash "$checker" "$scratch/canonical-valid" >/dev/null || { + echo "FAIL: model policy rejected a canonical managed-launch template" >&2 + exit 1 +} +canonical_pinned_output="$scratch/canonical-pinned.out" +if bash "$checker" "$scratch/canonical-pinned" >"$canonical_pinned_output" 2>&1; then + echo "FAIL: model policy accepted an account pin in a canonical managed-launch template" >&2 + exit 1 +fi +grep -Fq 'durably pins an account' "$canonical_pinned_output" || { + echo "FAIL: canonical template account pin lacked the expected diagnostic" >&2 + cat "$canonical_pinned_output" >&2 + exit 1 +} + +echo "PASS: canonical managed-launch templates are counted and durable account pins fail" diff --git a/bin/check-model-policy.sh b/bin/check-model-policy.sh index 150b42d7..f2b53ed0 100755 --- a/bin/check-model-policy.sh +++ b/bin/check-model-policy.sh @@ -29,6 +29,10 @@ while IFS=: read -r file line text; do if [[ "$file" == *.kdl ]]; then code="${code%%//*}" fi + if [[ "$file" == *.kdl.template && "$code" == *argv* && "$code" == *'"agent" "launch"'* ]]; then + code="${code//\"/}" + code="${code/agent launch/axe agent launch}" + fi remaining="$code" remaining="${remaining//--harness claude/--harness claude_harness}" @@ -108,9 +112,14 @@ while IFS=: read -r file line text; do axe_remaining="$axe_after" done done < <( - rg --no-ignore -n --no-heading \ - 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|axe[[:space:]]+agent[[:space:]]+launch' \ - "$scan_root" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true + { + rg --no-ignore -n --no-heading \ + 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|axe[[:space:]]+agent[[:space:]]+launch' \ + "$scan_root" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true + rg --no-ignore -n --no-heading \ + '"agent"[[:space:]]+"launch"' \ + "$scan_root" -g '*.kdl.template' || true + } ) [ "$launches" -gt 0 ] || fail "no maintained Claude or Codex launch sites were found" diff --git a/bin/corpus-inventory.sh b/bin/corpus-inventory.sh index 2ee412f8..e9c2d01a 100755 --- a/bin/corpus-inventory.sh +++ b/bin/corpus-inventory.sh @@ -56,19 +56,28 @@ for cell in "${cells[@]}"; do if [[ "$file" == *.kdl ]]; then code="${code%%//*}" fi - if [[ "$code" =~ exec[[:space:]]+claude([[:space:]]|$) ]] || - [[ "$code" =~ (^|[^[:alnum:]_-])claude[[:space:]]+- ]] || - [[ "$code" =~ exec[[:space:]]+axe[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+claude([[:space:]]|$) ]]; then + scan_code="$code" + if [[ "$file" == *.kdl.template ]]; then + scan_code="${code//\"/}" + fi + if [[ "$scan_code" =~ exec[[:space:]]+claude([[:space:]]|$) ]] || + [[ "$scan_code" =~ (^|[^[:alnum:]_-])claude[[:space:]]+- ]] || + [[ "$scan_code" =~ (exec[[:space:]]+axe|argv[[:space:]]+[^[:space:]]+)[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+claude([[:space:]]|$) ]]; then ((claude += 1)) - elif [[ "$code" =~ exec[[:space:]]+codex([[:space:]]|$) ]] || - [[ "$code" =~ (^|[^[:alnum:]_-])codex[[:space:]]+- ]] || - [[ "$code" =~ exec[[:space:]]+axe[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+codex([[:space:]]|$) ]]; then + elif [[ "$scan_code" =~ exec[[:space:]]+codex([[:space:]]|$) ]] || + [[ "$scan_code" =~ (^|[^[:alnum:]_-])codex[[:space:]]+- ]] || + [[ "$scan_code" =~ (exec[[:space:]]+axe|argv[[:space:]]+[^[:space:]]+)[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+codex([[:space:]]|$) ]]; then ((codex += 1)) fi done < <( - rg -n --no-heading \ - 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|exec[[:space:]]+axe[[:space:]]+agent[[:space:]]+launch' \ - "$cell_dir" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true + { + rg -n --no-heading \ + 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|axe[[:space:]]+agent[[:space:]]+launch' \ + "$cell_dir" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true + rg -n --no-heading \ + '"agent"[[:space:]]+"launch"' \ + "$cell_dir" -g '*.kdl.template' || true + } ) seats=$((claude + codex)) diff --git a/cells/agent-new-interview/README.md b/cells/agent-new-interview/README.md index 3dea4fea..b7cf84d5 100644 --- a/cells/agent-new-interview/README.md +++ b/cells/agent-new-interview/README.md @@ -28,27 +28,21 @@ Run it only with explicit paid-run authorization: st2 eval ./cells/agent-new-interview/ ``` -## Current composition gate - -This proposed cell intentionally uses the production typed launch boundary. -The current `st2 eval` team loader still boots `team { agent { command ... } }` -seats as transient commands; it does not materialize those seats as canonical -Agent Specs in the temporary catalog. - -Current Axe admission requires: - -1. `CATALOG`, `ST_ROOT`, and `PTY_ROOT` to agree on the st2 eval catalog; and -2. `ST_AGENT` to resolve to exactly one canonical Agent Spec. - -With the eval roots supplied, admission reaches the second check and rejects -`new.interviewer` because the temporary catalog contains zero canonical Agent -Specs for that identity. Therefore this paid cell currently fails at boot -before account selection or provider launch. Tokenlens can independently -return an eligible Claude account, so lack of pool quota is not the blocker. - -The principled fix belongs at the st2 eval/Axe composition seam: eval team -seats should be represented by canonical Agent Specs (or eval should directly -execute a canonical fleet declaration) and export the catalog-derived roots. -This cell must not work around the gap by fabricating an Agent Spec from inside -the seat, pinning an account in KDL, or launching bare Claude with ambient -credentials. +## Canonical eval seam + +The cell copies its fixture, deterministically publishes one canonical +`agents/evalhost/interviewer/agent.kdl` from the active runtime profile, and +then opts into st2's `canonical-agents` admission. The profile supplies the +exact immutable Axe adapter, absolute runtime-profile path, and canonical +generalist prompt. The declaration itself owns the typed launch trajectory and +contains no account pin. + +This makes the paid cell environment-bound by design: the selected runtime +profile is an explicit test prerequisite and part of the run evidence. The +cell is not hermetic across different profile artifacts. + +st2 validates and materializes that declaration before launch, routes the +kickoff through its canonical inbox, and requires a fresh interviewer reply +before the singleton eval can complete. The paid judgment run remains held +until the host's Claude workspace-trust projection has independent evidence; +the model-free bundle cell does not prove provider readiness. diff --git a/cells/agent-new-interview/agent-new-interview.kdl b/cells/agent-new-interview/agent-new-interview.kdl index e6887b25..1f94eca1 100644 --- a/cells/agent-new-interview/agent-new-interview.kdl +++ b/cells/agent-new-interview/agent-new-interview.kdl @@ -1,20 +1,17 @@ // One short human request is classified by a constrained temporary interviewer. // The interviewer emits semantic intent only; the fixture's deterministic submit // boundary validates that intent and renders the final B2 bundle. -team "new" { - agent "interviewer" { - workspace "./interviewer" - env { ST_AGENT "new.interviewer" } - command #"exec axe agent launch --harness claude --persona generalist --model claude-sonnet-5 --effort medium --mode managed-unattended --boot managed-v1"# - ding - } -} +host "evalhost" eval { copy "./fixture" + run "publish-canonical-interviewer" { + command "bash ./publish-interviewer.sh" + } + canonical-agents message { from "requester" - to "new.interviewer" + to "evalhost.interviewer" content "./task.md" } max-timeout "300s" diff --git a/cells/agent-new-interview/fixture/_templates/bus.st2.md b/cells/agent-new-interview/fixture/_templates/bus.st2.md new file mode 100644 index 00000000..f99a4448 --- /dev/null +++ b/cells/agent-new-interview/fixture/_templates/bus.st2.md @@ -0,0 +1,7 @@ +# st2 message bus + +Use the native st2 message bus for coordination. + +- Read your inbox with `st2 message inbox "$ST_AGENT"`. +- Send message bodies through stdin with a quoted heredoc delimiter. +- Reply to the requester after the bounded task is complete. diff --git a/cells/agent-new-interview/fixture/agent.kdl.template b/cells/agent-new-interview/fixture/agent.kdl.template new file mode 100644 index 00000000..57274912 --- /dev/null +++ b/cells/agent-new-interview/fixture/agent.kdl.template @@ -0,0 +1,28 @@ +agent "interviewer" { + identity "interviewer" + host "evalhost" + workspace "$CATALOG/interviewer" + + restart { + attempts 3 + interval "60s" + delay "0s" + mode "delay" + } + + env { + AGENT_LAUNCH_HOSTED "1" + AGENT_PERSONA "generalist" + AGENT_RUNTIME_PROFILE "@PROFILE@" + ST_AGENT "evalhost.interviewer" + } + + argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--model" "claude-sonnet-5" "--effort" "medium" "--persona" "generalist" "--mode" "managed-unattended" "--boot" "managed-v1" + ding + + render { + copy "_templates/generalist.md" ".st2/PERSONA.md" + copy "_templates/bus.st2.md" ".st2/bus.md" + git-exclude ".st2/" + } +} diff --git a/cells/agent-new-interview/fixture/publish-interviewer.sh b/cells/agent-new-interview/fixture/publish-interviewer.sh new file mode 100755 index 00000000..a5534a8b --- /dev/null +++ b/cells/agent-new-interview/fixture/publish-interviewer.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +profile="${AGENT_RUNTIME_PROFILE:-$HOME/.config/coding-agents/profile.json}" +case "$profile" in + /*) ;; + *) + echo "runtime profile must be absolute: $profile" >&2 + exit 1 + ;; +esac + +adapter="$(jq -er '.agentSpec.adapterBin | select(type == "string" and startswith("/"))' "$profile")" +persona_source="$(jq -er '.personas.prompts.generalist | select(type == "string" and startswith("/"))' "$profile")" +for path in "$profile" "$adapter" "$persona_source"; do + case "$path" in + *[!A-Za-z0-9_./+-]*) + echo "runtime path cannot be represented in canonical KDL: $path" >&2 + exit 1 + ;; + esac +done + +mkdir -p ./agents/evalhost/interviewer ./_templates +cp "$persona_source" ./_templates/generalist.md +sed \ + -e "s|@PROFILE@|$profile|" \ + -e "s|@ADAPTER@|$adapter|" \ + ./agent.kdl.template >./agents/evalhost/interviewer/agent.kdl diff --git a/cells/canonical-agent-runtime-smoke/README.md b/cells/canonical-agent-runtime-smoke/README.md new file mode 100644 index 00000000..18ef2fd0 --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/README.md @@ -0,0 +1,13 @@ +# canonical-agent-runtime-smoke + +This model-free cell exercises st2's production Agent Spec eval seam with one +deterministic shell agent. The canonical declaration is the sole launch and +routing authority: st2 boots it, delivers the kickoff to its native inbox, +requires a fresh reply, grades the root and routing receipts, and tears the PTY +down through normal eval cleanup. + +It uses no Axe, account, model, or provider harness. + +```sh +st2 eval ./cells/canonical-agent-runtime-smoke/ +``` diff --git a/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl b/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl new file mode 100644 index 00000000..9562a142 --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl @@ -0,0 +1,21 @@ +host "evalhost" + +eval { + copy "./fixture" + canonical-agents + message { + from "requester" + to "evalhost.probe" + content "./task.md" + } + max-timeout "30s" + + judges { + judge "CANONICAL ROOTS — the shell agent receives only the eval-owned catalog, bus, and PTY roots" { + exec "test -f $CATALOG/probe/roots-ok" + } + judge "CANONICAL KICKOFF — the request arrives in the admitted Agent Spec inbox before reply" { + exec "test -f $CATALOG/probe/kickoff-seen" + } + } +} diff --git a/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl b/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl new file mode 100644 index 00000000..bb962859 --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl @@ -0,0 +1,9 @@ +agent "probe" { + identity "probe" + host "evalhost" + workspace "$CATALOG" + env { + ST_AGENT "evalhost.probe" + } + argv "bash" "$CATALOG/probe.sh" +} diff --git a/cells/canonical-agent-runtime-smoke/fixture/probe.sh b/cells/canonical-agent-runtime-smoke/fixture/probe.sh new file mode 100755 index 00000000..6ef660a6 --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/fixture/probe.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +test "$CATALOG" = "$ST_ROOT" +test "$PTY_ROOT" = "$CATALOG/pty" +test "$ST_AGENT" = "evalhost.probe" +mkdir -p "$CATALOG/probe" +touch "$CATALOG/probe/roots-ok" + +for _ in $(seq 1 200); do + if test "$(st2 message ls "$ST_AGENT" --catalog "$CATALOG" --from requester --count)" -gt 0; then + touch "$CATALOG/probe/kickoff-seen" + st2 message send requester --catalog "$CATALOG" --as "$ST_AGENT" <<'MSG' +Canonical kickoff received. +MSG + exec sleep 60 + fi + sleep 0.05 +done + +echo "canonical kickoff did not arrive" >&2 +exit 1 diff --git a/cells/canonical-agent-runtime-smoke/task.md b/cells/canonical-agent-runtime-smoke/task.md new file mode 100644 index 00000000..c1084de8 --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/task.md @@ -0,0 +1 @@ +Confirm the canonical eval runtime path. diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index 3cbb8a52..3e513919 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -107,14 +107,24 @@ one-sentence human request account per run. Corpus inventory, model policy, lifecycle, and harness checks recognize this typed launch as a paid model seat and reject durable account pins. -- **Current composition gate:** `st2 eval` currently boots legacy team - `command` seats without creating a canonical Agent Spec for the temporary - identity. Axe correctly rejects that seat before account selection because - `ST_AGENT` resolves to zero canonical specs. This is a blocked paid - experiment, not evidence against account selection or interviewer judgment. - The accepted direction is to make eval seats canonical at the runtime seam; - a cell-local compatibility wrapper, fabricated spec, account pin, or ambient - bare-provider launch is outside the contract. +- The paid cell uses st2's explicit `canonical-agents` seam. A deterministic + pre-admission run reads the active runtime profile and publishes one + canonical interviewer declaration with its exact immutable Axe adapter, + absolute profile path, and canonical persona source. st2 then carries that + declaration unchanged through strict validation, materialization, launch, + kickoff routing, singleton completion, and teardown. No compact eval seat, + compatibility wrapper, account pin, or ambient provider launch participates. +- The paid cell is environment-bound to that explicit runtime-profile + artifact; the artifact identity belongs in run evidence and results are not + represented as hermetic across different profiles. +- The canonical runtime seam removes the prior zero-Agent-Spec admission + blocker. The paid judgment run remains held until the host's Claude + workspace-trust projection is independently proven; model-free evidence + does not establish provider readiness. +- The model-free `canonical-agent-runtime-smoke` cell independently proves the + runtime seam with one deterministic shell Agent Spec: strict admission, + boot, canonical kickoff, fresh reply, and normal teardown, with no Axe, + account, model, or provider harness. - Passing model-free evidence proves the bundle boundary and its negative controls only. The autonomous-selection claim remains unaccepted until the paid judgment cell has a structured PASS receipt. diff --git a/evidence/harness-exclusions.tsv b/evidence/harness-exclusions.tsv index 0961c818..f785a70e 100644 --- a/evidence/harness-exclusions.tsv +++ b/evidence/harness-exclusions.tsv @@ -1,5 +1,6 @@ cell surface reason agent-new-bundle-contract cell deterministic semantic-intent lowering and mutation probe; no harness seat +canonical-agent-runtime-smoke cell deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness seat context-resource-continuity cell deterministic native context/resource restart probe; no harness seat docs judge:cold-reader one-shot offline Claude print grader; no bus identity, DING, or hook surface exec-lifecycle-logging cell deterministic native exec lifecycle probe; no harness seat From 44b75cfc052d3d21530a3d5a366441162efb2feb Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:59:28 +0200 Subject: [PATCH 05/12] evals: harden canonical session evidence agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- AGENT-SPEC.md | 8 ++-- CATALOG.md | 2 +- README.md | 2 + bin/check-agent-new-renderer-security.sh | 30 +++++++++++++ ...check-canonical-seat-template-mutations.sh | 32 ++++++++++++++ bin/check-canonical-seat-template.sh | 44 +++++++++++++++++++ bin/check-corpus.sh | 18 ++++---- bin/check-event-first.sh | 19 +++++--- bin/check-harness-contract.sh | 32 +++++++++++--- bin/check-preflight-safety.sh | 10 +++++ bin/check-st2-pin-consistency.sh | 20 +++++++++ bin/corpus-inventory.sh | 43 ++++-------------- bin/model-seat-inventory.sh | 42 +++++++++++++++--- bin/st2-pin.sh | 6 +++ cells/agent-new-bundle-contract/README.md | 5 ++- .../agent-new-bundle-contract.kdl | 8 +++- .../fixture/inputs/workspace-injection.json | 18 ++++++++ .../fixture/render-intent.sh | 2 +- .../fixture/interviewer/render-intent.sh | 2 +- docs/vrs/spec.md | 9 ++++ 20 files changed, 280 insertions(+), 72 deletions(-) create mode 100755 bin/check-agent-new-renderer-security.sh create mode 100755 bin/check-canonical-seat-template-mutations.sh create mode 100755 bin/check-canonical-seat-template.sh create mode 100755 bin/check-st2-pin-consistency.sh create mode 100755 bin/st2-pin.sh create mode 100644 cells/agent-new-bundle-contract/fixture/inputs/workspace-injection.json diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 2b5bda16..1c46d640 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -432,8 +432,8 @@ Inspect the declaration, every referenced template, and every workspace destinat materialization command. Materialization is byte-idempotent and does not imply hook installation. Starting the network is a separate, explicitly authorized action. -For source `9887b28`, the accepted Linux executable has SHA256 -`d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d`; its published archive has SHA256 -`32ee103bd17ccb3e155ac63d816a3906c2470a3c98e3cc04b56e5a67138b9927`. `bin/check-corpus.sh` verifies -the variable-age version contract, exact installed binary, embedded full source commit, strict semantic +For source `0ec6a22e877f9374260202c5f76dabc4579fc7e3`, the accepted local-source Linux executable has +SHA256 `df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db`. +`bin/check-st2-pin-consistency.sh` keeps this active spec, the README, and executable preflight aligned. +`bin/check-corpus.sh` verifies the variable-age version contract, exact installed binary, strict semantic validation, fixture resets, and the rest of the model-free corpus gate before an eval may run. diff --git a/CATALOG.md b/CATALOG.md index 8df7a740..2bcb19c6 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -23,7 +23,7 @@ harness-native loader plus canonical hook file. | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `ding-reply` | Claude | `claude-sonnet-5` / medium | 1 | low | `420s` | 2 | — | **NO STRUCTURED RUN** | -| `docs` | Claude | `claude-sonnet-5` / medium | 3 | high | `1200s` | 5 | — | **NO STRUCTURED RUN** | +| `docs` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 5 | — | **NO STRUCTURED RUN** | | `exec-lifecycle-logging` | model-free | — | 0 | none | `90s` | 5 | — | **NO STRUCTURED RUN** | | `feature-fit` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 5 | — | **NO STRUCTURED RUN** | | `fork-in-the-road` | Claude | `claude-sonnet-5` / medium | 4 | high | `1200s` | 5 | — | **NO STRUCTURED RUN** | diff --git a/README.md b/README.md index 717030d0..eeaf54a3 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,8 @@ VERDICT: PASS Requirements are `st2 0.1.0` from source [`0ec6a22`](https://github.com/compoundingtech/st2/commit/0ec6a22e877f9374260202c5f76dabc4579fc7e3), +using the accepted local-source Linux binary SHA256 +`df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db`; `pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and Node for JavaScript fixtures. A paid cell also needs every harness named by its dry-run row. diff --git a/bin/check-agent-new-renderer-security.sh b/bin/check-agent-new-renderer-security.sh new file mode 100755 index 00000000..718eace4 --- /dev/null +++ b/bin/check-agent-new-renderer-security.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +input="$repo_root/cells/agent-new-bundle-contract/fixture/inputs/workspace-injection.json" +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +renderers=( + "$repo_root/cells/agent-new-bundle-contract/fixture/render-intent.sh" + "$repo_root/cells/agent-new-interview/fixture/interviewer/render-intent.sh" +) + +for renderer in "${renderers[@]}"; do + name="$(basename "$(dirname "$renderer")")" + out="$scratch/$name" + if bash "$renderer" "$input" "$out" >/dev/null 2>&1; then + echo "FAIL: $renderer accepted a workspace KDL injection" >&2 + exit 1 + fi + [ ! -e "$out/agents" ] || { + echo "FAIL: $renderer emitted an Agent Spec after rejecting workspace KDL injection" >&2 + exit 1 + } +done + +echo "PASS: both Agent Spec renderers reject workspace KDL injection before output" diff --git a/bin/check-canonical-seat-template-mutations.sh b/bin/check-canonical-seat-template-mutations.sh new file mode 100755 index 00000000..8aecf918 --- /dev/null +++ b/bin/check-canonical-seat-template-mutations.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +checker="$repo_root/bin/check-canonical-seat-template.sh" +source_template="$repo_root/cells/agent-new-interview/fixture/agent.kdl.template" +source_publisher="$repo_root/cells/agent-new-interview/fixture/publish-interviewer.sh" +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +cp "$source_template" "$scratch/valid.template" +cp "$source_publisher" "$scratch/publisher.sh" +"$checker" "$scratch/valid.template" "$scratch/publisher.sh" >/dev/null + +cp "$source_template" "$scratch/missing-boot.template" +sed -i 's/"--boot" "managed-v1"//' "$scratch/missing-boot.template" +if "$checker" "$scratch/missing-boot.template" "$scratch/publisher.sh" >/dev/null 2>&1; then + echo "FAIL: canonical template without managed-v1 boot passed" >&2 + exit 1 +fi + +cp "$source_template" "$scratch/missing-overlay.template" +sed -i '/copy "_templates\/bus.st2.md" ".st2\/bus.md"/d' "$scratch/missing-overlay.template" +if "$checker" "$scratch/missing-overlay.template" "$scratch/publisher.sh" >/dev/null 2>&1; then + echo "FAIL: canonical template without bus overlay passed" >&2 + exit 1 +fi + +echo "PASS: removing a required launch axis or canonical overlay fails the template gate" diff --git a/bin/check-canonical-seat-template.sh b/bin/check-canonical-seat-template.sh new file mode 100755 index 00000000..418c7119 --- /dev/null +++ b/bin/check-canonical-seat-template.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +template="${1:?canonical Agent Spec template required}" +publisher="${2:?canonical Agent Spec publisher required}" + +require_once() { + local pattern="$1" description="$2" + if [ "$(grep -Fxc "$pattern" "$template" || true)" -ne 1 ]; then + echo "FAIL: $template must contain exactly one $description" >&2 + exit 1 + fi +} + +require_once ' AGENT_LAUNCH_HOSTED "1"' "hosted launch marker" +require_once ' AGENT_PERSONA "generalist"' "canonical persona" +require_once ' AGENT_RUNTIME_PROFILE "@PROFILE@"' "runtime-profile projection" +require_once ' argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--model" "claude-sonnet-5" "--effort" "medium" "--persona" "generalist" "--mode" "managed-unattended" "--boot" "managed-v1"' "typed managed-v1 launch" +require_once ' copy "_templates/generalist.md" ".st2/PERSONA.md"' "canonical persona overlay" +require_once ' copy "_templates/bus.st2.md" ".st2/bus.md"' "canonical bus overlay" +require_once ' git-exclude ".st2/"' "overlay exclusion" +require_once ' ding' "native DING declaration" + +if grep -Fq -- '--account' "$template"; then + echo "FAIL: $template durably pins an account" >&2 + exit 1 +fi + +for projection in \ + 'profile="${AGENT_RUNTIME_PROFILE:-$HOME/.config/coding-agents/profile.json}"' \ + 'case "$profile" in' \ + '/*) ;;' \ + '.agentSpec.adapterBin' \ + '.personas.prompts.generalist' \ + 's|@PROFILE@|$profile|' \ + 's|@ADAPTER@|$adapter|' \ + 'cp "$persona_source" ./_templates/generalist.md'; do + grep -Fq "$projection" "$publisher" || { + echo "FAIL: $publisher omits canonical projection $projection" >&2 + exit 1 + } +done + +echo "PASS: $template is an account-neutral managed-v1 canonical template with explicit profile and overlay projection" diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index 0d9684c3..66572986 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -5,22 +5,20 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -expected_source="0ec6a22" -expected_source_full="0ec6a22e877f9374260202c5f76dabc4579fc7e3" -expected_binary_sha256="df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db" -expected_version_regex='^st2 0\.1\.0 — running from local source \(0ec6a22, .+ ago\)$' +source bin/st2-pin.sh +bin/check-st2-pin-consistency.sh st2_path="$(command -v st2)" actual_version="$(st2 --version)" -[[ "$actual_version" =~ $expected_version_regex ]] || { - echo "FAIL: expected st2 0.1.0 from pinned source $expected_source, found $actual_version" >&2 +[[ "$actual_version" =~ $ST2_VERSION_REGEX ]] || { + echo "FAIL: expected st2 0.1.0 from pinned source $ST2_SOURCE_SHORT, found $actual_version" >&2 exit 1 } actual_binary_sha256="$(sha256sum "$st2_path" | awk '{ print $1 }')" -[ "$actual_binary_sha256" = "$expected_binary_sha256" ] || { - echo "FAIL: expected st2 binary sha256 $expected_binary_sha256, found $actual_binary_sha256 at $st2_path" >&2 +[ "$actual_binary_sha256" = "$ST2_BINARY_SHA256" ] || { + echo "FAIL: expected st2 binary sha256 $ST2_BINARY_SHA256, found $actual_binary_sha256 at $st2_path" >&2 exit 1 } -echo "PASS: pinned runner source $expected_source_full ($actual_version; sha256 $actual_binary_sha256)" +echo "PASS: pinned runner source $ST2_SOURCE_FULL ($actual_version; sha256 $actual_binary_sha256)" mapfile -d '' shell_files < <( find bin cells -type f -name '*.sh' -not -path '*/_git/*' -print0 | sort -z @@ -29,8 +27,10 @@ bash -n "${shell_files[@]}" echo "PASS: ${#shell_files[@]} shell files parse" bin/check-preflight-safety.sh +bin/check-agent-new-renderer-security.sh bin/check-model-policy.sh bin/check-model-policy-mutations.sh +bin/check-canonical-seat-template-mutations.sh bin/check-run-history.sh bin/check-retired-surfaces.sh bin/model-seat-inventory.sh >/dev/null diff --git a/bin/check-event-first.sh b/bin/check-event-first.sh index d5c9702b..ef449c29 100755 --- a/bin/check-event-first.sh +++ b/bin/check-event-first.sh @@ -29,19 +29,26 @@ required=( "report completion or blockers over the st2 bus" ) -while IFS=$'\t' read -r cell agent _harness _workspace _st_agent command_line; do - kdl="cells/$cell/$cell.kdl" - command_text="$(sed -n "${command_line}p" "$kdl")" +while IFS=$'\t' read -r cell agent _harness _workspace _st_agent source_kind source_path source_line; do + command_text="$(sed -n "${source_line}p" "$source_path")" + if [ "$source_kind" = "canonical-template" ]; then + grep -Fq '"--mode" "managed-unattended"' <<< "$command_text" || + fail "$source_path:$source_line agent $agent canonical launch omits managed-unattended mode" + grep -Fq '"--boot" "managed-v1"' <<< "$command_text" || + fail "$source_path:$source_line agent $agent canonical launch omits managed-v1 event-first boot contract" + continue + fi + kdl="$source_path" if grep -Fq 'exec axe agent launch ' <<< "$command_text"; then grep -Fq -- '--mode managed-unattended' <<< "$command_text" || - fail "$kdl:$command_line agent $agent Axe launch omits managed-unattended mode" + fail "$kdl:$source_line agent $agent Axe launch omits managed-unattended mode" grep -Fq -- '--boot managed-v1' <<< "$command_text" || - fail "$kdl:$command_line agent $agent Axe launch omits managed-v1 event-first boot contract" + fail "$kdl:$source_line agent $agent Axe launch omits managed-v1 event-first boot contract" continue fi for phrase in "${required[@]}"; do grep -Fiq "$phrase" <<< "$command_text" || - fail "$kdl:$command_line agent $agent does not teach '$phrase'" + fail "$kdl:$source_line agent $agent does not teach '$phrase'" done done < "$inventory" diff --git a/bin/check-harness-contract.sh b/bin/check-harness-contract.sh index 41ae40a4..bf335de1 100755 --- a/bin/check-harness-contract.sh +++ b/bin/check-harness-contract.sh @@ -6,14 +6,19 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" selected="all" +selected_cell="" if [ "$#" -eq 2 ] && [ "$1" = "--harness" ]; then selected="$2" case "$selected" in Claude|Codex) ;; *) echo "usage: bin/check-harness-contract.sh [--harness Claude|Codex]" >&2; exit 2 ;; esac +elif [ "$#" -eq 2 ] && [ "$1" = "--cell" ]; then + selected_cell="$2" + [ -d "cells/$selected_cell" ] || + { echo "FAIL: unknown cell $selected_cell" >&2; exit 2; } elif [ "$#" -ne 0 ]; then - echo "usage: bin/check-harness-contract.sh [--harness Claude|Codex]" >&2 + echo "usage: bin/check-harness-contract.sh [--harness Claude|Codex] | [--cell CELL]" >&2 exit 2 fi @@ -79,7 +84,10 @@ prepare_cell() { declare -A roots=() failed=0 checked=0 -while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do +while IFS=$'\t' read -r cell agent harness workspace st_agent source_kind source_path source_line; do + if [ -n "$selected_cell" ] && [ "$cell" != "$selected_cell" ]; then + continue + fi if [ "$selected" != "all" ] && [ "$harness" != "$selected" ]; then continue fi @@ -92,7 +100,11 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do } fi - relative="${workspace#./}" + if [[ "$workspace" == "\$CATALOG/"* ]]; then + relative="${workspace#\$CATALOG/}" + else + relative="${workspace#./}" + fi target="${roots[$cell]}/$relative" [ -n "$relative" ] || target="${roots[$cell]}" if [ ! -d "$target" ]; then @@ -101,10 +113,18 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do continue fi - kdl="cells/$cell/$cell.kdl" - command_text="$(sed -n "${command_line}p" "$kdl")" + command_text="$(sed -n "${source_line}p" "$source_path")" axe_launch=0 - if grep -Fq 'exec axe agent launch ' <<< "$command_text"; then + if [ "$source_kind" = "canonical-template" ]; then + publisher="$(dirname "$source_path")/publish-interviewer.sh" + bin/check-canonical-seat-template.sh "$source_path" "$publisher" >/dev/null || { + failed=1 + continue + } + [ -s "$(dirname "$source_path")/_templates/bus.st2.md" ] || + { echo "FAIL: $cell/$agent canonical bus overlay source is missing" >&2; failed=1; } + axe_launch=1 + elif grep -Fq 'exec axe agent launch ' <<< "$command_text"; then axe_launch=1 grep -Fq -- '--mode managed-unattended' <<< "$command_text" || { echo "FAIL: $cell/$agent Axe launch omits managed-unattended mode" >&2; failed=1; } diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index 4af8f41e..61745445 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -7,6 +7,9 @@ cd "$repo_root" reachable=( bin/check-corpus.sh + bin/check-agent-new-renderer-security.sh + bin/check-canonical-seat-template-mutations.sh + bin/check-canonical-seat-template.sh bin/check-event-first.sh bin/check-fixture-reset.sh bin/check-fixture-reset-terminal.sh @@ -20,6 +23,7 @@ reachable=( bin/check-preflight-safety.sh bin/check-run-history.sh bin/check-retired-surfaces.sh + bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh @@ -27,8 +31,11 @@ reachable=( bin/corpus-inventory.sh bin/generate-catalog.sh bin/model-seat-inventory.sh + bin/st2-pin.sh ) materializers=( + cells/agent-new-bundle-contract/fixture/render-intent.sh + cells/agent-new-interview/fixture/interviewer/render-intent.sh cells/signal-rename/fixture/materialize.sh cells/signal-rename-codex/fixture/materialize.sh cells/weird-git-setup/fixture/setup-megarepo.sh @@ -56,6 +63,8 @@ mapfile -t direct < <( LC_ALL=C sort -u ) expected_direct=( + bin/check-agent-new-renderer-security.sh + bin/check-canonical-seat-template-mutations.sh bin/check-event-first.sh bin/check-fixture-reset-terminal.sh bin/check-harness-contract.sh @@ -68,6 +77,7 @@ expected_direct=( bin/check-preflight-safety.sh bin/check-retired-surfaces.sh bin/check-run-history.sh + bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh diff --git a/bin/check-st2-pin-consistency.sh b/bin/check-st2-pin-consistency.sh new file mode 100755 index 00000000..6a276983 --- /dev/null +++ b/bin/check-st2-pin-consistency.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +source bin/st2-pin.sh + +grep -Fq "$ST2_SOURCE_FULL" AGENT-SPEC.md +grep -Fq "$ST2_SOURCE_FULL" README.md +grep -Fq "$ST2_BINARY_SHA256" AGENT-SPEC.md +grep -Fq "$ST2_BINARY_SHA256" README.md +grep -Fq 'source bin/st2-pin.sh' bin/check-corpus.sh + +if rg -n '9887b28|d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d' \ + AGENT-SPEC.md README.md bin/check-corpus.sh bin/st2-pin.sh; then + echo "FAIL: active st2 pin surfaces retain the superseded source or binary hash" >&2 + exit 1 +fi + +echo "PASS: active spec, README, and executable preflight share st2 $ST2_SOURCE_FULL / $ST2_BINARY_SHA256" diff --git a/bin/corpus-inventory.sh b/bin/corpus-inventory.sh index e9c2d01a..7fdd67b9 100755 --- a/bin/corpus-inventory.sh +++ b/bin/corpus-inventory.sh @@ -13,6 +13,13 @@ elif [ "$#" -ne 0 ]; then exit 2 fi +seat_inventory="$(mktemp)" +cleanup() { + rm -f -- "$seat_inventory" +} +trap cleanup EXIT +bin/model-seat-inventory.sh --no-header > "$seat_inventory" + if [ "$include_header" -eq 1 ]; then printf 'cell\tharness\tmodels\teffort\tmodel_seats\tcost_band\ttimeout\theld_out_judges\n' fi @@ -45,40 +52,8 @@ for cell in "${cells[@]}"; do } timeout="${timeouts[0]}" - claude=0 - codex=0 - while IFS=: read -r file line text; do - trimmed="${text#"${text%%[![:space:]]*}"}" - if [[ "$trimmed" == \#* || "$trimmed" == //* ]]; then - continue - fi - code="$text" - if [[ "$file" == *.kdl ]]; then - code="${code%%//*}" - fi - scan_code="$code" - if [[ "$file" == *.kdl.template ]]; then - scan_code="${code//\"/}" - fi - if [[ "$scan_code" =~ exec[[:space:]]+claude([[:space:]]|$) ]] || - [[ "$scan_code" =~ (^|[^[:alnum:]_-])claude[[:space:]]+- ]] || - [[ "$scan_code" =~ (exec[[:space:]]+axe|argv[[:space:]]+[^[:space:]]+)[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+claude([[:space:]]|$) ]]; then - ((claude += 1)) - elif [[ "$scan_code" =~ exec[[:space:]]+codex([[:space:]]|$) ]] || - [[ "$scan_code" =~ (^|[^[:alnum:]_-])codex[[:space:]]+- ]] || - [[ "$scan_code" =~ (exec[[:space:]]+axe|argv[[:space:]]+[^[:space:]]+)[[:space:]]+agent[[:space:]]+launch[[:space:]].*--harness[[:space:]]+codex([[:space:]]|$) ]]; then - ((codex += 1)) - fi - done < <( - { - rg -n --no-heading \ - 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|axe[[:space:]]+agent[[:space:]]+launch' \ - "$cell_dir" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true - rg -n --no-heading \ - '"agent"[[:space:]]+"launch"' \ - "$cell_dir" -g '*.kdl.template' || true - } - ) + claude="$(awk -F '\t' -v cell="$cell" '$1 == cell && $3 == "Claude" { count += 1 } END { print count + 0 }' "$seat_inventory")" + codex="$(awk -F '\t' -v cell="$cell" '$1 == cell && $3 == "Codex" { count += 1 } END { print count + 0 }' "$seat_inventory")" seats=$((claude + codex)) if [ "$claude" -gt 0 ] && [ "$codex" -gt 0 ]; then diff --git a/bin/model-seat-inventory.sh b/bin/model-seat-inventory.sh index b3ba6919..bf1dc538 100755 --- a/bin/model-seat-inventory.sh +++ b/bin/model-seat-inventory.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Emit every maintained model seat directly from the root folder-eval declarations. +# Emit every maintained model seat from compact eval declarations and canonical templates. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -28,7 +28,7 @@ while IFS= read -r cell; do continue } - if ! awk -v cell="$cell" ' + if ! awk -v cell="$cell" -v source_path="$kdl" ' function direct(line) { return index(line, child_indent) == 1 } @@ -49,7 +49,7 @@ while IFS= read -r cell; do bad = 1 } if (workspace != "" && st_agent != "" && ding == 1) { - printf "%s\t%s\t%s\t%s\t%s\t%d\n", cell, agent, harness, workspace, st_agent, command_line + printf "%s\t%s\t%s\t%s\t%s\tcompact\t%s\t%d\n", cell, agent, harness, workspace, st_agent, source_path, command_line } } /^[[:space:]]*agent[[:space:]]+"/ { @@ -125,9 +125,39 @@ while IFS= read -r cell; do fi done < <(find cells -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | LC_ALL=C sort) +while IFS= read -r template; do + cell="${template#cells/}" + cell="${cell%%/*}" + agent="$(sed -n 's/^[[:space:]]*agent[[:space:]]*"\([^"]*\)".*/\1/p' "$template")" + workspace="$(sed -n 's/^[[:space:]]*workspace[[:space:]]*"\([^"]*\)".*/\1/p' "$template")" + st_agent="$(sed -n 's/^[[:space:]]*ST_AGENT[[:space:]]*"\([^"]*\)".*/\1/p' "$template")" + command_line="$(rg -n '^[[:space:]]*argv[[:space:]].*"agent"[[:space:]]+"launch"' "$template" | cut -d: -f1)" + command_text="$(sed -n "${command_line}p" "$template")" + ding_count="$(rg -c '^[[:space:]]*ding[[:space:]]*$' "$template" || true)" + harness="" + if grep -Fq '"--harness" "claude"' <<< "$command_text"; then + harness="Claude" + elif grep -Fq '"--harness" "codex"' <<< "$command_text"; then + harness="Codex" + fi + if [ -z "$agent" ] || [ -z "$workspace" ] || [ -z "$st_agent" ] || + [ -z "$command_line" ] || [ -z "$harness" ] || [ "$ding_count" -ne 1 ]; then + echo "FAIL: canonical model-seat template is incomplete: $template" >&2 + failed=1 + continue + fi + printf '%s\t%s\t%s\t%s\t%s\tcanonical-template\t%s\t%s\n' \ + "$cell" "$agent" "$harness" "$workspace" "$st_agent" "$template" "$command_line" \ + >> "$inventory" +done < <(find cells -type f -name 'agent.kdl.template' | LC_ALL=C sort) + launches="$( - rg -n --no-heading '^[[:space:]]*command[[:space:]]+.*exec ((claude|codex)[[:space:]]|axe agent launch[[:space:]])' \ - cells/*/*.kdl | wc -l | tr -d ' ' + { + rg -n --no-heading '^[[:space:]]*command[[:space:]]+.*exec ((claude|codex)[[:space:]]|axe agent launch[[:space:]])' \ + cells/*/*.kdl || true + rg -n --no-heading '^[[:space:]]*argv[[:space:]].*"agent"[[:space:]]+"launch"' \ + cells -g 'agent.kdl.template' || true + } | wc -l | tr -d ' ' )" rows="$(wc -l < "$inventory" | tr -d ' ')" if [ "$rows" -ne "$launches" ]; then @@ -137,6 +167,6 @@ fi [ "$failed" -eq 0 ] || exit 1 if [ "$include_header" -eq 1 ]; then - printf 'cell\tagent\tharness\tworkspace\tst_agent\tcommand_line\n' + printf 'cell\tagent\tharness\tworkspace\tst_agent\tsource_kind\tsource_path\tsource_line\n' fi cat "$inventory" diff --git a/bin/st2-pin.sh b/bin/st2-pin.sh new file mode 100755 index 00000000..1784a70f --- /dev/null +++ b/bin/st2-pin.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +ST2_SOURCE_SHORT="0ec6a22" +ST2_SOURCE_FULL="0ec6a22e877f9374260202c5f76dabc4579fc7e3" +ST2_BINARY_SHA256="df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db" +ST2_VERSION_REGEX='^st2 0\.1\.0 — running from local source \(0ec6a22, .+ ago\)$' diff --git a/cells/agent-new-bundle-contract/README.md b/cells/agent-new-bundle-contract/README.md index bd0adaa1..f8aaa065 100644 --- a/cells/agent-new-bundle-contract/README.md +++ b/cells/agent-new-bundle-contract/README.md @@ -5,9 +5,10 @@ interview handoff. A small semantic `axe.agent-creation-intent.v1` record is the only interviewer output. The renderer owns canonical Agent Spec KDL and the initial inbox Resource. -Two valid one-session intents exercise the same lowering path. Two invalid +Two valid one-session intents exercise the same lowering path. Three invalid inputs prove the input boundary is closed: an interviewer cannot pin an -account or introduce an unsupported trajectory value. +account, introduce an unsupported trajectory value, or inject KDL through the +workspace path. Held-out mutation checks copy valid output and independently prove that the bundle grader rejects: diff --git a/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl b/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl index 392e71e6..bd141e7f 100644 --- a/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl +++ b/cells/agent-new-bundle-contract/agent-new-bundle-contract.kdl @@ -18,6 +18,10 @@ eval { allow-nonzero command "bash ./render-intent.sh ./inputs/trajectory-drift.json ./out/trajectory-drift" } + run "workspace-injection" { + allow-nonzero + command "bash ./render-intent.sh ./inputs/workspace-injection.json ./out/workspace-injection" + } judges { judge "IMPLEMENTATION — typed intent lowers to the exact canonical managed trajectory" { @@ -29,8 +33,8 @@ eval { judge "INITIAL CONTEXT — goal and references become an inbox Resource, not KDL fields" { exec "bash ./judges/grade.sh inbox" } - judge "CLOSED INPUT — account pins, unknown fields, and unstable trajectory values fail closed" { - exec #"test "$RUN_forbidden_account_EXIT" -ne 0 && test "$RUN_trajectory_drift_EXIT" -ne 0"# + judge "CLOSED INPUT — account pins, unstable trajectory values, and KDL workspace injection fail closed" { + exec #"test "$RUN_forbidden_account_EXIT" -ne 0 && test "$RUN_trajectory_drift_EXIT" -ne 0 && test "$RUN_workspace_injection_EXIT" -ne 0 && test ! -e "$CATALOG/out/workspace-injection/agents""# } judge "NEGATIVE CONTROLS — raw provider argv, trajectory drift, and missing context are rejected" { exec "bash ./judges/mutation-check.sh" diff --git a/cells/agent-new-bundle-contract/fixture/inputs/workspace-injection.json b/cells/agent-new-bundle-contract/fixture/inputs/workspace-injection.json new file mode 100644 index 00000000..36a10eaa --- /dev/null +++ b/cells/agent-new-bundle-contract/fixture/inputs/workspace-injection.json @@ -0,0 +1,18 @@ +{ + "schema": "axe.agent-creation-intent.v1", + "decision": "commit", + "identity": "dotfiles.axe.issue-40.implementation", + "workspace": "/workspace/dotfiles\"\n argv \"sh\" \"-c\" \"touch /tmp/injected\"", + "goal": "Implement Axe issue 40.", + "references": [ + "https://github.com/compoundingtech/axe/issues/40" + ], + "trajectory": { + "harness": "codex", + "model": "gpt-5.6-sol", + "effort": "high", + "persona": "generalist", + "mode": "managed-unattended", + "boot": "managed-v1" + } +} diff --git a/cells/agent-new-bundle-contract/fixture/render-intent.sh b/cells/agent-new-bundle-contract/fixture/render-intent.sh index e28b2777..812df71f 100755 --- a/cells/agent-new-bundle-contract/fixture/render-intent.sh +++ b/cells/agent-new-bundle-contract/fixture/render-intent.sh @@ -12,7 +12,7 @@ jq -e ' .schema == "axe.agent-creation-intent.v1" and .decision == "commit" and (.identity | test("^[a-z0-9][a-z0-9.-]*[a-z0-9]$")) and - (.workspace | test("^/")) and + (.workspace | test("^/[-A-Za-z0-9._/+:]+$")) and (.goal | type == "string" and length > 0 and test("[\"\\n\\r]") | not) and (.references | type == "array" and length > 0 and all(.[]; type == "string" and test("^https://github\\.com/"))) and diff --git a/cells/agent-new-interview/fixture/interviewer/render-intent.sh b/cells/agent-new-interview/fixture/interviewer/render-intent.sh index d84b651f..a8f1cf33 100755 --- a/cells/agent-new-interview/fixture/interviewer/render-intent.sh +++ b/cells/agent-new-interview/fixture/interviewer/render-intent.sh @@ -12,7 +12,7 @@ jq -e ' .schema == "axe.agent-creation-intent.v1" and .decision == "commit" and (.identity | test("^[a-z0-9][a-z0-9.-]*[a-z0-9]$")) and - (.workspace | test("^/")) and + (.workspace | test("^/[-A-Za-z0-9._/+:]+$")) and (.goal | type == "string" and length > 0 and test("[\"\\n\\r]") | not) and (.references | type == "array" and length > 0 and all(.[]; type == "string" and test("^https://github\\.com/"))) and diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index 3e513919..c198e269 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -101,6 +101,10 @@ one-sentence human request explicit in that intent; the deterministic boundary owns canonical KDL, keeps account selection out of durable state, and places the goal and references in an initial inbox Resource. +- The renderer accepts workspace values only from a closed absolute-path + alphabet and rejects invalid input before creating an Agent Spec. A + model-free malicious-workspace control gates both renderers against KDL + injection. - The temporary interviewer itself launches through typed `axe agent launch` with explicit harness, model, effort, persona, mode, and boot axes. The cell does not pin `--account`: Axe selects an eligible @@ -117,6 +121,11 @@ one-sentence human request - The paid cell is environment-bound to that explicit runtime-profile artifact; the artifact identity belongs in run evidence and results are not represented as hermetic across different profiles. +- One structured seat inventory is authoritative for both compact eval seats + and canonical paid templates. Corpus cost classification, event-first + policy, and harness-overlay checks consume its source-kind/path records; + canonical template mutations prove required launch axes and overlays are + non-vacuous. - The canonical runtime seam removes the prior zero-Agent-Spec admission blocker. The paid judgment run remains held until the host's Claude workspace-trust projection is independently proven; model-free evidence From 024f0ad54671c378b85f47722b12299c667e9801 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:14:30 +0200 Subject: [PATCH 06/12] evals: prepare canonical Git workspace agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- CATALOG.md | 2 +- bin/check-agent-new-interview-attempts.sh | 49 +++++++++++++++++++ bin/check-corpus.sh | 1 + bin/check-harness-contract.sh | 11 ++++- bin/check-preflight-safety.sh | 4 ++ cells/agent-new-interview/README.md | 11 +++++ .../agent-new-interview.kdl | 3 ++ .../fixture/prepare-interviewer-worktree.sh | 24 +++++++++ cells/canonical-agent-runtime-smoke/README.md | 5 +- .../canonical-agent-runtime-smoke.kdl | 9 ++++ .../fixture/_templates/bus.st2.md | 1 + .../fixture/_templates/generalist.md | 1 + .../fixture/agents/evalhost/probe/agent.kdl | 7 ++- .../fixture/prepare-interviewer-worktree.sh | 18 +++++++ docs/vrs/spec.md | 26 ++++++---- evidence/agent-new-interview-attempts.tsv | 2 + 16 files changed, 161 insertions(+), 13 deletions(-) create mode 100755 bin/check-agent-new-interview-attempts.sh create mode 100755 cells/agent-new-interview/fixture/prepare-interviewer-worktree.sh create mode 100644 cells/canonical-agent-runtime-smoke/fixture/_templates/bus.st2.md create mode 100644 cells/canonical-agent-runtime-smoke/fixture/_templates/generalist.md create mode 100755 cells/canonical-agent-runtime-smoke/fixture/prepare-interviewer-worktree.sh create mode 100644 evidence/agent-new-interview-attempts.tsv diff --git a/CATALOG.md b/CATALOG.md index 2bcb19c6..ac7a8592 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -18,7 +18,7 @@ harness-native loader plus canonical hook file. |---|---|---|---:|---|---|---:|---|---| | `agent-new-bundle-contract` | model-free | — | 0 | none | `60s` | 5 | — | **NO STRUCTURED RUN** | | `agent-new-interview` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | -| `canonical-agent-runtime-smoke` | model-free | — | 0 | none | `30s` | 2 | — | **NO STRUCTURED RUN** | +| `canonical-agent-runtime-smoke` | model-free | — | 0 | none | `30s` | 4 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | diff --git a/bin/check-agent-new-interview-attempts.sh b/bin/check-agent-new-interview-attempts.sh new file mode 100755 index 00000000..c970c5a4 --- /dev/null +++ b/bin/check-agent-new-interview-attempts.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Validate append-only pre-run evidence without representing it as a judged model run. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +ledger="evidence/agent-new-interview-attempts.tsv" +expected_header=$'recorded_at_utc\tsource_commit\tst2_commit\tst2_sha256\taxe_feature_head\tadapter\truntime_profile\tstage\tresult\tdiagnostic\tcleanup' + +[ -f "$ledger" ] && [ "$(head -n 1 "$ledger")" = "$expected_header" ] || { + echo "FAIL: $ledger is missing or has an unexpected header" >&2 + exit 1 +} + +if git cat-file -e "HEAD:$ledger" 2>/dev/null; then + prior="$(mktemp)" + trap 'rm -f -- "$prior"' EXIT + git show "HEAD:$ledger" >"$prior" + prior_size="$(wc -c <"$prior")" + current_size="$(wc -c <"$ledger")" + [ "$current_size" -ge "$prior_size" ] && cmp -n "$prior_size" "$prior" "$ledger" || { + echo "FAIL: $ledger rewrites or removes existing evidence; only append rows" >&2 + exit 1 + } +fi + +rows=0 +while IFS=$'\t' read -r recorded source st2_commit st2_sha axe_head adapter profile stage result diagnostic cleanup extra; do + [ "$recorded" != "recorded_at_utc" ] || continue + [ -n "$recorded" ] || continue + [ -z "${extra:-}" ] || { echo "FAIL: $ledger row has extra columns" >&2; exit 1; } + [[ "$recorded" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || + { echo "FAIL: invalid recorded_at_utc $recorded" >&2; exit 1; } + [[ "$source" =~ ^[0-9a-f]{40}$ ]] && git cat-file -e "$source^{commit}" 2>/dev/null || + { echo "FAIL: unavailable source commit $source" >&2; exit 1; } + [[ "$st2_commit" =~ ^[0-9a-f]{40}$ && "$st2_sha" =~ ^[0-9a-f]{64}$ ]] || + { echo "FAIL: st2 identity is not exact" >&2; exit 1; } + [[ "$axe_head" =~ ^[0-9a-f]{7,40}$ && "$adapter" == /nix/store/* && "$profile" == /nix/store/* ]] || + { echo "FAIL: Axe/runtime artifact identity is not exact" >&2; exit 1; } + [ "$stage" = "materialization" ] && [ "$result" = "PRE_AXE_FAIL" ] || + { echo "FAIL: pre-run evidence claims an unsupported stage or result" >&2; exit 1; } + [ -n "$diagnostic" ] && [ "$cleanup" = "catalog-absent;process-absent" ] || + { echo "FAIL: pre-run evidence lacks diagnostic or cleanup proof" >&2; exit 1; } + ((rows += 1)) +done <"$ledger" + +[ "$rows" -gt 0 ] || { echo "FAIL: $ledger has no evidence rows" >&2; exit 1; } +printf 'PASS: %d append-only pre-Axe attempt row(s) validated with exact artifacts and cleanup\n' "$rows" diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index 66572986..ebefc6c7 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -32,6 +32,7 @@ bin/check-model-policy.sh bin/check-model-policy-mutations.sh bin/check-canonical-seat-template-mutations.sh bin/check-run-history.sh +bin/check-agent-new-interview-attempts.sh bin/check-retired-surfaces.sh bin/model-seat-inventory.sh >/dev/null bin/check-event-first.sh diff --git a/bin/check-harness-contract.sh b/bin/check-harness-contract.sh index bf335de1..af061b96 100755 --- a/bin/check-harness-contract.sh +++ b/bin/check-harness-contract.sh @@ -56,6 +56,9 @@ prepare_cell() { } cp -a "$fixture"/. "$root"/ case "$cell" in + agent-new-interview) + script="$root/prepare-interviewer-worktree.sh" + ;; signal-rename|signal-rename-codex) script="$root/materialize.sh" ;; @@ -78,7 +81,11 @@ prepare_cell() { echo "FAIL: $cell materializer contains a provider/reconcile/network command" >&2 return 1 fi - CATALOG="$root" bash "$script" >/dev/null + if [ "$cell" = "agent-new-interview" ]; then + CATALOG="$root" bash "$script" "$root/interviewer" >/dev/null + else + CATALOG="$root" bash "$script" >/dev/null + fi } declare -A roots=() @@ -123,6 +130,8 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent source_kind source } [ -s "$(dirname "$source_path")/_templates/bus.st2.md" ] || { echo "FAIL: $cell/$agent canonical bus overlay source is missing" >&2; failed=1; } + git -C "$target" rev-parse --is-inside-work-tree | grep -Fxq true || + { echo "FAIL: $cell/$agent canonical workspace is not a Git worktree" >&2; failed=1; } axe_launch=1 elif grep -Fq 'exec axe agent launch ' <<< "$command_text"; then axe_launch=1 diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index 61745445..e94646f8 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -8,6 +8,7 @@ cd "$repo_root" reachable=( bin/check-corpus.sh bin/check-agent-new-renderer-security.sh + bin/check-agent-new-interview-attempts.sh bin/check-canonical-seat-template-mutations.sh bin/check-canonical-seat-template.sh bin/check-event-first.sh @@ -34,8 +35,10 @@ reachable=( bin/st2-pin.sh ) materializers=( + cells/agent-new-interview/fixture/prepare-interviewer-worktree.sh cells/agent-new-bundle-contract/fixture/render-intent.sh cells/agent-new-interview/fixture/interviewer/render-intent.sh + cells/canonical-agent-runtime-smoke/fixture/prepare-interviewer-worktree.sh cells/signal-rename/fixture/materialize.sh cells/signal-rename-codex/fixture/materialize.sh cells/weird-git-setup/fixture/setup-megarepo.sh @@ -63,6 +66,7 @@ mapfile -t direct < <( LC_ALL=C sort -u ) expected_direct=( + bin/check-agent-new-interview-attempts.sh bin/check-agent-new-renderer-security.sh bin/check-canonical-seat-template-mutations.sh bin/check-event-first.sh diff --git a/cells/agent-new-interview/README.md b/cells/agent-new-interview/README.md index b7cf84d5..786ba2f9 100644 --- a/cells/agent-new-interview/README.md +++ b/cells/agent-new-interview/README.md @@ -46,3 +46,14 @@ kickoff through its canonical inbox, and requires a fresh interviewer reply before the singleton eval can complete. The paid judgment run remains held until the host's Claude workspace-trust projection has independent evidence; the model-free bundle cell does not prove provider readiness. + +The fixture setup creates the copied interviewer directory as a deterministic, +clean Git worktree before canonical admission. This is required because the +canonical overlay retains `git-exclude ".st2/"`, and materialization warnings +remain fatal. + +The first authorized attempt is recorded in +[`evidence/agent-new-interview-attempts.tsv`](../../evidence/agent-new-interview-attempts.tsv). +It stopped at materialization before Axe, account selection, provider +readiness, or judges because the earlier fixture lacked that Git-worktree +setup. Cleanup removed the temporary catalog and process. diff --git a/cells/agent-new-interview/agent-new-interview.kdl b/cells/agent-new-interview/agent-new-interview.kdl index 1f94eca1..73ef088c 100644 --- a/cells/agent-new-interview/agent-new-interview.kdl +++ b/cells/agent-new-interview/agent-new-interview.kdl @@ -5,6 +5,9 @@ host "evalhost" eval { copy "./fixture" + run "prepare-interviewer-worktree" { + command "bash ./prepare-interviewer-worktree.sh" + } run "publish-canonical-interviewer" { command "bash ./publish-interviewer.sh" } diff --git a/cells/agent-new-interview/fixture/prepare-interviewer-worktree.sh b/cells/agent-new-interview/fixture/prepare-interviewer-worktree.sh new file mode 100755 index 00000000..9d9abdeb --- /dev/null +++ b/cells/agent-new-interview/fixture/prepare-interviewer-worktree.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +workspace="${1:-./interviewer}" +[ -d "$workspace" ] || { + echo "interviewer workspace does not exist: $workspace" >&2 + exit 1 +} +[ ! -e "$workspace/.git" ] || { + echo "interviewer workspace is already a Git worktree: $workspace" >&2 + exit 1 +} + +git -C "$workspace" init -q --initial-branch=main +git -C "$workspace" add -A +GIT_AUTHOR_DATE="2000-01-01T00:00:00Z" \ +GIT_COMMITTER_DATE="2000-01-01T00:00:00Z" \ + git -C "$workspace" \ + -c core.hooksPath=/dev/null \ + -c user.name=eval-fixture \ + -c user.email=eval-fixture@local \ + commit -q -m "fixture: canonical interviewer workspace" + +test -z "$(git -C "$workspace" status --porcelain=v1)" diff --git a/cells/canonical-agent-runtime-smoke/README.md b/cells/canonical-agent-runtime-smoke/README.md index 18ef2fd0..ee035337 100644 --- a/cells/canonical-agent-runtime-smoke/README.md +++ b/cells/canonical-agent-runtime-smoke/README.md @@ -6,7 +6,10 @@ routing authority: st2 boots it, delivers the kickoff to its native inbox, requires a fresh reply, grades the root and routing receipts, and tears the PTY down through normal eval cleanup. -It uses no Axe, account, model, or provider harness. +Before admission, a deterministic run creates the same disposable Git-worktree +shape required by the paid interviewer. Strict materialization copies the +canonical persona and bus overlays and applies `git-exclude` with zero +warnings. It uses no Axe, account, model, or provider harness. ```sh st2 eval ./cells/canonical-agent-runtime-smoke/ diff --git a/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl b/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl index 9562a142..809b7f97 100644 --- a/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl +++ b/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl @@ -2,6 +2,9 @@ host "evalhost" eval { copy "./fixture" + run "prepare-interviewer-worktree" { + command "bash ./prepare-interviewer-worktree.sh" + } canonical-agents message { from "requester" @@ -17,5 +20,11 @@ eval { judge "CANONICAL KICKOFF — the request arrives in the admitted Agent Spec inbox before reply" { exec "test -f $CATALOG/probe/kickoff-seen" } + judge "STRICT MATERIALIZATION — canonical persona and bus overlays materialize into a clean Git worktree" { + exec #"test -s "$CATALOG/interviewer/.st2/PERSONA.md" && test -s "$CATALOG/interviewer/.st2/bus.md" && grep -Fxq ".st2/" "$CATALOG/interviewer/.git/info/exclude" && test -z "$(git -C "$CATALOG/interviewer" status --porcelain=v1)""# + } + judge "MODEL FREE — the admitted Agent Spec contains no provider, account, or Axe launch" { + exec #"! grep -Eq 'axe|claude|codex|opencode|pi|--account' "$CATALOG/agents/evalhost/probe/agent.kdl""# + } } } diff --git a/cells/canonical-agent-runtime-smoke/fixture/_templates/bus.st2.md b/cells/canonical-agent-runtime-smoke/fixture/_templates/bus.st2.md new file mode 100644 index 00000000..663bbbfd --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/fixture/_templates/bus.st2.md @@ -0,0 +1 @@ +Synthetic canonical st2 bus instructions for model-free materialization. diff --git a/cells/canonical-agent-runtime-smoke/fixture/_templates/generalist.md b/cells/canonical-agent-runtime-smoke/fixture/_templates/generalist.md new file mode 100644 index 00000000..998fa71d --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/fixture/_templates/generalist.md @@ -0,0 +1 @@ +Synthetic canonical generalist prompt for model-free materialization. diff --git a/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl b/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl index bb962859..426428c7 100644 --- a/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl +++ b/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl @@ -1,9 +1,14 @@ agent "probe" { identity "probe" host "evalhost" - workspace "$CATALOG" + workspace "$CATALOG/interviewer" env { ST_AGENT "evalhost.probe" } argv "bash" "$CATALOG/probe.sh" + render { + copy "_templates/generalist.md" ".st2/PERSONA.md" + copy "_templates/bus.st2.md" ".st2/bus.md" + git-exclude ".st2/" + } } diff --git a/cells/canonical-agent-runtime-smoke/fixture/prepare-interviewer-worktree.sh b/cells/canonical-agent-runtime-smoke/fixture/prepare-interviewer-worktree.sh new file mode 100755 index 00000000..b4e9db80 --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/fixture/prepare-interviewer-worktree.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +workspace="./interviewer" +mkdir -p "$workspace" +printf '%s\n' "canonical materialization fixture" > "$workspace/README.md" + +git -C "$workspace" init -q --initial-branch=main +git -C "$workspace" add README.md +GIT_AUTHOR_DATE="2000-01-01T00:00:00Z" \ +GIT_COMMITTER_DATE="2000-01-01T00:00:00Z" \ + git -C "$workspace" \ + -c core.hooksPath=/dev/null \ + -c user.name=eval-fixture \ + -c user.email=eval-fixture@local \ + commit -q -m "fixture: canonical materialization workspace" + +test -z "$(git -C "$workspace" status --porcelain=v1)" diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index c198e269..180d4607 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -112,12 +112,14 @@ one-sentence human request checks recognize this typed launch as a paid model seat and reject durable account pins. - The paid cell uses st2's explicit `canonical-agents` seam. A deterministic - pre-admission run reads the active runtime profile and publishes one - canonical interviewer declaration with its exact immutable Axe adapter, - absolute profile path, and canonical persona source. st2 then carries that - declaration unchanged through strict validation, materialization, launch, - kickoff routing, singleton completion, and teardown. No compact eval seat, - compatibility wrapper, account pin, or ambient provider launch participates. + pre-admission run first creates the copied interviewer workspace as a clean, + deterministic Git worktree. A second run reads the active runtime profile + and publishes one canonical interviewer declaration with its exact immutable + Axe adapter, absolute profile path, and canonical persona source. st2 then + carries that declaration unchanged through strict validation, + warning-free materialization, launch, kickoff routing, singleton completion, + and teardown. No compact eval seat, compatibility wrapper, account pin, or + ambient provider launch participates. - The paid cell is environment-bound to that explicit runtime-profile artifact; the artifact identity belongs in run evidence and results are not represented as hermetic across different profiles. @@ -131,9 +133,15 @@ one-sentence human request workspace-trust projection is independently proven; model-free evidence does not establish provider readiness. - The model-free `canonical-agent-runtime-smoke` cell independently proves the - runtime seam with one deterministic shell Agent Spec: strict admission, - boot, canonical kickoff, fresh reply, and normal teardown, with no Axe, - account, model, or provider harness. + runtime seam with one deterministic shell Agent Spec: the same disposable + Git-worktree shape, strict persona/bus overlay materialization with zero + warnings, boot, canonical kickoff, fresh reply, and normal teardown, with no + Axe, account, model, or provider harness. +- The first authorized paid attempt is evidence only for a pre-Axe fixture + defect: required `git-exclude` materialization failed because the copied + workspace was not a Git worktree. It reached no account selection, provider + readiness, trust behavior, judge, or verdict and left no catalog or process + residue. It cannot be represented as provider evidence. - Passing model-free evidence proves the bundle boundary and its negative controls only. The autonomous-selection claim remains unaccepted until the paid judgment cell has a structured PASS receipt. diff --git a/evidence/agent-new-interview-attempts.tsv b/evidence/agent-new-interview-attempts.tsv new file mode 100644 index 00000000..c7e386e1 --- /dev/null +++ b/evidence/agent-new-interview-attempts.tsv @@ -0,0 +1,2 @@ +recorded_at_utc source_commit st2_commit st2_sha256 axe_feature_head adapter runtime_profile stage result diagnostic cleanup +2026-07-30T20:06:02Z 44b75cfc052d3d21530a3d5a366441162efb2feb 0ec6a22e877f9374260202c5f76dabc4579fc7e3 df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db c1bbce7f7 /nix/store/jyvz9050ba3wv214k862wnkvd03yxll6-axe/bin/axe /nix/store/5lp3nnsgb591qhhk5cp44ffajmnqcihy-coding-agent-runtime-profile/share/coding-agents/profile.json materialization PRE_AXE_FAIL interviewer workspace was not a Git worktree, so required git-exclude produced a fatal warning catalog-absent;process-absent From d72e3280d9db80720eae8aaa36b81468a60e123c Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:24:02 +0200 Subject: [PATCH 07/12] evals: harden pre-Axe evidence agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- ...-agent-new-interview-attempts-mutations.sh | 33 +++++++++++++++++ bin/check-agent-new-interview-attempts.sh | 32 +++++++++------- bin/check-append-only-file.sh | 37 +++++++++++++++++++ bin/check-corpus.sh | 16 +++++++- bin/check-preflight-safety.sh | 3 ++ cells/canonical-agent-runtime-smoke/README.md | 4 ++ evidence/agent-new-interview-attempts.tsv | 4 +- 7 files changed, 112 insertions(+), 17 deletions(-) create mode 100755 bin/check-agent-new-interview-attempts-mutations.sh create mode 100755 bin/check-append-only-file.sh diff --git a/bin/check-agent-new-interview-attempts-mutations.sh b/bin/check-agent-new-interview-attempts-mutations.sh new file mode 100755 index 00000000..b6b577e6 --- /dev/null +++ b/bin/check-agent-new-interview-attempts-mutations.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Prove the append-only helper accepts an append and rejects a baseline-row rewrite. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +scratch="$(mktemp -d)" +trap 'rm -rf -- "$scratch"' EXIT + +git -C "$scratch" init -q --initial-branch=main +mkdir -p "$scratch/evidence" +printf 'header\nbaseline-row\n' >"$scratch/evidence/attempts.tsv" +git -C "$scratch" add evidence/attempts.tsv +GIT_AUTHOR_DATE="2000-01-01T00:00:00Z" \ +GIT_COMMITTER_DATE="2000-01-01T00:00:00Z" \ + git -C "$scratch" \ + -c core.hooksPath=/dev/null \ + -c user.name=eval-fixture \ + -c user.email=eval-fixture@local \ + commit -q -m "fixture: append-only baseline" +baseline="$(git -C "$scratch" rev-parse HEAD)" + +printf 'appended-row\n' >>"$scratch/evidence/attempts.tsv" +"$repo_root/bin/check-append-only-file.sh" \ + --repo-root "$scratch" --baseline "$baseline" --path evidence/attempts.tsv >/dev/null + +printf 'header\nrewritten-row\nappended-row\n' >"$scratch/evidence/attempts.tsv" +if "$repo_root/bin/check-append-only-file.sh" \ + --repo-root "$scratch" --baseline "$baseline" --path evidence/attempts.tsv >/dev/null 2>&1; then + echo "FAIL: append-only helper accepted a rewritten baseline row" >&2 + exit 1 +fi + +echo "PASS: append-only mutation accepts appended bytes and rejects a baseline-row rewrite" diff --git a/bin/check-agent-new-interview-attempts.sh b/bin/check-agent-new-interview-attempts.sh index c970c5a4..04cce282 100755 --- a/bin/check-agent-new-interview-attempts.sh +++ b/bin/check-agent-new-interview-attempts.sh @@ -1,28 +1,28 @@ #!/usr/bin/env bash -# Validate append-only pre-run evidence without representing it as a judged model run. +# Validate pre-run evidence without representing it as a judged model run. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" ledger="evidence/agent-new-interview-attempts.tsv" -expected_header=$'recorded_at_utc\tsource_commit\tst2_commit\tst2_sha256\taxe_feature_head\tadapter\truntime_profile\tstage\tresult\tdiagnostic\tcleanup' +baseline="" +if [ "$#" -eq 2 ] && [ "$1" = "--baseline" ]; then + baseline="$2" +elif [ "$#" -ne 0 ]; then + echo "usage: $0 [--baseline COMMIT]" >&2 + exit 2 +fi +expected_header=$'recorded_at_utc\tsource_commit\tst2_commit\tst2_sha256\taxe_source_commit\tadapter\truntime_profile\tstage\tresult\tdiagnostic\tcleanup' [ -f "$ledger" ] && [ "$(head -n 1 "$ledger")" = "$expected_header" ] || { echo "FAIL: $ledger is missing or has an unexpected header" >&2 exit 1 } -if git cat-file -e "HEAD:$ledger" 2>/dev/null; then - prior="$(mktemp)" - trap 'rm -f -- "$prior"' EXIT - git show "HEAD:$ledger" >"$prior" - prior_size="$(wc -c <"$prior")" - current_size="$(wc -c <"$ledger")" - [ "$current_size" -ge "$prior_size" ] && cmp -n "$prior_size" "$prior" "$ledger" || { - echo "FAIL: $ledger rewrites or removes existing evidence; only append rows" >&2 - exit 1 - } +if [ -n "$baseline" ]; then + bin/check-append-only-file.sh \ + --repo-root "$repo_root" --baseline "$baseline" --path "$ledger" fi rows=0 @@ -36,7 +36,7 @@ while IFS=$'\t' read -r recorded source st2_commit st2_sha axe_head adapter prof { echo "FAIL: unavailable source commit $source" >&2; exit 1; } [[ "$st2_commit" =~ ^[0-9a-f]{40}$ && "$st2_sha" =~ ^[0-9a-f]{64}$ ]] || { echo "FAIL: st2 identity is not exact" >&2; exit 1; } - [[ "$axe_head" =~ ^[0-9a-f]{7,40}$ && "$adapter" == /nix/store/* && "$profile" == /nix/store/* ]] || + [[ "$axe_head" =~ ^[0-9a-f]{40}$ && "$adapter" == /nix/store/* && "$profile" == /nix/store/* ]] || { echo "FAIL: Axe/runtime artifact identity is not exact" >&2; exit 1; } [ "$stage" = "materialization" ] && [ "$result" = "PRE_AXE_FAIL" ] || { echo "FAIL: pre-run evidence claims an unsupported stage or result" >&2; exit 1; } @@ -46,4 +46,8 @@ while IFS=$'\t' read -r recorded source st2_commit st2_sha axe_head adapter prof done <"$ledger" [ "$rows" -gt 0 ] || { echo "FAIL: $ledger has no evidence rows" >&2; exit 1; } -printf 'PASS: %d append-only pre-Axe attempt row(s) validated with exact artifacts and cleanup\n' "$rows" +if [ -n "$baseline" ]; then + printf 'PASS: %d pre-Axe attempt row(s) validated with exact artifacts, cleanup, and an immutable baseline\n' "$rows" +else + printf 'PASS: %d pre-Axe attempt row(s) validated with exact artifacts and cleanup; no append-only baseline supplied\n' "$rows" +fi diff --git a/bin/check-append-only-file.sh b/bin/check-append-only-file.sh new file mode 100755 index 00000000..a2737391 --- /dev/null +++ b/bin/check-append-only-file.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Prove that a working-tree file preserves an immutable baseline blob as an exact prefix. +set -euo pipefail + +repo_root="" +baseline="" +path="" +while [ "$#" -gt 0 ]; do + case "$1" in + --repo-root) repo_root="${2:-}"; shift 2 ;; + --baseline) baseline="${2:-}"; shift 2 ;; + --path) path="${2:-}"; shift 2 ;; + *) echo "usage: $0 --repo-root DIR --baseline REF --path RELATIVE_PATH" >&2; exit 2 ;; + esac +done +[ -n "$repo_root" ] && [ -n "$baseline" ] && [ -n "$path" ] || + { echo "usage: $0 --repo-root DIR --baseline REF --path RELATIVE_PATH" >&2; exit 2; } +[[ "$path" != /* && "$path" != *..* ]] || + { echo "FAIL: append-only path must be repository-relative" >&2; exit 1; } + +baseline_commit="$(git -C "$repo_root" rev-parse --verify "$baseline^{commit}")" || + { echo "FAIL: append-only baseline is not an immutable commit: $baseline" >&2; exit 1; } +git -C "$repo_root" cat-file -e "$baseline_commit:$path" 2>/dev/null || + { echo "FAIL: baseline $baseline_commit has no $path to compare" >&2; exit 1; } +[ -f "$repo_root/$path" ] || { echo "FAIL: working file is missing: $path" >&2; exit 1; } + +prior="$(mktemp)" +trap 'rm -f -- "$prior"' EXIT +git -C "$repo_root" show "$baseline_commit:$path" >"$prior" +prior_size="$(wc -c <"$prior")" +current_size="$(wc -c <"$repo_root/$path")" +[ "$current_size" -ge "$prior_size" ] && + cmp -n "$prior_size" "$prior" "$repo_root/$path" || { + echo "FAIL: $path rewrites or removes baseline $baseline_commit; only append bytes" >&2 + exit 1 + } +printf 'PASS: %s preserves baseline %s as an exact prefix\n' "$path" "$baseline_commit" diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index ebefc6c7..5af09ab3 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -32,7 +32,21 @@ bin/check-model-policy.sh bin/check-model-policy-mutations.sh bin/check-canonical-seat-template-mutations.sh bin/check-run-history.sh -bin/check-agent-new-interview-attempts.sh +attempt_baseline="${EVALS_EVIDENCE_BASELINE_REF:-}" +if [ -z "$attempt_baseline" ] && git rev-parse --verify origin/main^{commit} >/dev/null 2>&1; then + candidate_baseline="$(git merge-base HEAD origin/main)" + current_commit="$(git rev-parse HEAD)" + if [ "$candidate_baseline" != "$current_commit" ] && + git cat-file -e "$candidate_baseline:evidence/agent-new-interview-attempts.tsv" 2>/dev/null; then + attempt_baseline="$candidate_baseline" + fi +fi +if [ -n "$attempt_baseline" ]; then + bin/check-agent-new-interview-attempts.sh --baseline "$attempt_baseline" +else + bin/check-agent-new-interview-attempts.sh +fi +bin/check-agent-new-interview-attempts-mutations.sh bin/check-retired-surfaces.sh bin/model-seat-inventory.sh >/dev/null bin/check-event-first.sh diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index e94646f8..1f71275e 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -7,6 +7,8 @@ cd "$repo_root" reachable=( bin/check-corpus.sh + bin/check-append-only-file.sh + bin/check-agent-new-interview-attempts-mutations.sh bin/check-agent-new-renderer-security.sh bin/check-agent-new-interview-attempts.sh bin/check-canonical-seat-template-mutations.sh @@ -66,6 +68,7 @@ mapfile -t direct < <( LC_ALL=C sort -u ) expected_direct=( + bin/check-agent-new-interview-attempts-mutations.sh bin/check-agent-new-interview-attempts.sh bin/check-agent-new-renderer-security.sh bin/check-canonical-seat-template-mutations.sh diff --git a/cells/canonical-agent-runtime-smoke/README.md b/cells/canonical-agent-runtime-smoke/README.md index ee035337..9ba5201b 100644 --- a/cells/canonical-agent-runtime-smoke/README.md +++ b/cells/canonical-agent-runtime-smoke/README.md @@ -11,6 +11,10 @@ shape required by the paid interviewer. Strict materialization copies the canonical persona and bus overlays and applies `git-exclude` with zero warnings. It uses no Axe, account, model, or provider harness. +Its six gating results are the successful preparation run step, canonical team +completion, and four declared judges. The run-step result is emitted and graded +by st2 in addition to the judges declared in KDL. + ```sh st2 eval ./cells/canonical-agent-runtime-smoke/ ``` diff --git a/evidence/agent-new-interview-attempts.tsv b/evidence/agent-new-interview-attempts.tsv index c7e386e1..30351938 100644 --- a/evidence/agent-new-interview-attempts.tsv +++ b/evidence/agent-new-interview-attempts.tsv @@ -1,2 +1,2 @@ -recorded_at_utc source_commit st2_commit st2_sha256 axe_feature_head adapter runtime_profile stage result diagnostic cleanup -2026-07-30T20:06:02Z 44b75cfc052d3d21530a3d5a366441162efb2feb 0ec6a22e877f9374260202c5f76dabc4579fc7e3 df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db c1bbce7f7 /nix/store/jyvz9050ba3wv214k862wnkvd03yxll6-axe/bin/axe /nix/store/5lp3nnsgb591qhhk5cp44ffajmnqcihy-coding-agent-runtime-profile/share/coding-agents/profile.json materialization PRE_AXE_FAIL interviewer workspace was not a Git worktree, so required git-exclude produced a fatal warning catalog-absent;process-absent +recorded_at_utc source_commit st2_commit st2_sha256 axe_source_commit adapter runtime_profile stage result diagnostic cleanup +2026-07-30T20:06:02Z 44b75cfc052d3d21530a3d5a366441162efb2feb 0ec6a22e877f9374260202c5f76dabc4579fc7e3 df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db c1bbce7f712f823f1df4447837d46e972910d9fa /nix/store/jyvz9050ba3wv214k862wnkvd03yxll6-axe/bin/axe /nix/store/5lp3nnsgb591qhhk5cp44ffajmnqcihy-coding-agent-runtime-profile/share/coding-agents/profile.json materialization PRE_AXE_FAIL interviewer workspace was not a Git worktree, so required git-exclude produced a fatal warning catalog-absent;process-absent From 0015922e8906178f02e74b71e773ec145efed665 Mon Sep 17 00:00:00 2001 From: Nathan Date: Thu, 30 Jul 2026 22:44:12 +0200 Subject: [PATCH 08/12] Adapt canonical eval smoke to merged st2 --- AGENT-SPEC.md | 8 +- CATALOG.md | 2 +- README.md | 4 +- bin/check-canonical-agent-runtime-smoke.sh | 74 +++++++++++++++++++ bin/check-st2-pin-consistency.sh | 3 +- bin/st2-pin.sh | 8 +- cells/canonical-agent-runtime-smoke/README.md | 19 +++-- .../canonical-agent-runtime-smoke.kdl | 13 +++- .../fleet/remote/declaration/agent.kdl | 8 ++ .../.managed/arbitrary/declaration}/agent.kdl | 11 ++- .../fixture/probe.sh | 2 + 11 files changed, 129 insertions(+), 23 deletions(-) create mode 100755 bin/check-canonical-agent-runtime-smoke.sh create mode 100644 cells/canonical-agent-runtime-smoke/fixture/fleet/remote/declaration/agent.kdl rename cells/canonical-agent-runtime-smoke/fixture/{agents/evalhost/probe => organization/.managed/arbitrary/declaration}/agent.kdl (52%) diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 1c46d640..6f38b50f 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -1,8 +1,8 @@ # Canonical st2 agent specification This is the sole agent-authoring specification for this repository. It is pinned to st2 -[`0ec6a22e877f9374260202c5f76dabc4579fc7e3`](https://github.com/compoundingtech/st2/commit/0ec6a22e877f9374260202c5f76dabc4579fc7e3) -(`0.1.0`, source `0ec6a22`). It documents the hand-authored KDL accepted at that commit. Do not infer +[`c6846f6239329f0803142afc06c15a07b93937c1`](https://github.com/compoundingtech/st2/commit/c6846f6239329f0803142afc06c15a07b93937c1) +(`0.1.0`, source `c6846f6`). It documents the hand-authored KDL accepted at that commit. Do not infer additional fields or commands from older corpus fixtures. st2 runs long-lived `service` agents made of interactive `pty` tasks and terminal-free `exec` tasks. @@ -432,8 +432,8 @@ Inspect the declaration, every referenced template, and every workspace destinat materialization command. Materialization is byte-idempotent and does not imply hook installation. Starting the network is a separate, explicitly authorized action. -For source `0ec6a22e877f9374260202c5f76dabc4579fc7e3`, the accepted local-source Linux executable has -SHA256 `df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db`. +For source `c6846f6239329f0803142afc06c15a07b93937c1`, the accepted local-source Linux executable has +SHA256 `2bba8d58be24250bc262f75f835ce2d780369add275774f3f2135c623d23d29c`. `bin/check-st2-pin-consistency.sh` keeps this active spec, the README, and executable preflight aligned. `bin/check-corpus.sh` verifies the variable-age version contract, exact installed binary, strict semantic validation, fixture resets, and the rest of the model-free corpus gate before an eval may run. diff --git a/CATALOG.md b/CATALOG.md index ac7a8592..916011c0 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -18,7 +18,7 @@ harness-native loader plus canonical hook file. |---|---|---|---:|---|---|---:|---|---| | `agent-new-bundle-contract` | model-free | — | 0 | none | `60s` | 5 | — | **NO STRUCTURED RUN** | | `agent-new-interview` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | -| `canonical-agent-runtime-smoke` | model-free | — | 0 | none | `30s` | 4 | — | **NO STRUCTURED RUN** | +| `canonical-agent-runtime-smoke` | model-free | — | 0 | none | `30s` | 7 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | diff --git a/README.md b/README.md index eeaf54a3..1e84f970 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,9 @@ VERDICT: PASS ``` Requirements are `st2 0.1.0` from source -[`0ec6a22`](https://github.com/compoundingtech/st2/commit/0ec6a22e877f9374260202c5f76dabc4579fc7e3), +[`c6846f6`](https://github.com/compoundingtech/st2/commit/c6846f6239329f0803142afc06c15a07b93937c1), using the accepted local-source Linux binary SHA256 -`df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db`; +`2bba8d58be24250bc262f75f835ce2d780369add275774f3f2135c623d23d29c`; `pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and Node for JavaScript fixtures. A paid cell also needs every harness named by its dry-run row. diff --git a/bin/check-canonical-agent-runtime-smoke.sh b/bin/check-canonical-agent-runtime-smoke.sh new file mode 100755 index 00000000..af6ec31f --- /dev/null +++ b/bin/check-canonical-agent-runtime-smoke.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +source bin/st2-pin.sh + +st2_bin="$(command -v st2)" +version="$(st2 --version)" +sha256="$(sha256sum "$st2_bin" | awk '{ print $1 }')" +[[ "$version" =~ $ST2_VERSION_REGEX ]] || { + echo "FAIL: expected st2 source $ST2_SOURCE_SHORT, found $version" >&2 + exit 1 +} +[ "$sha256" = "$ST2_BINARY_SHA256" ] || { + echo "FAIL: expected st2 binary $ST2_BINARY_SHA256, found $sha256 at $st2_bin" >&2 + exit 1 +} + +bin/corpus-inventory.sh --no-header | + awk -F '\t' '$1 == "canonical-agent-runtime-smoke" && $2 == "model-free" && $5 == 0 { found = 1 } END { exit !found }' || { + echo "FAIL: canonical-agent-runtime-smoke is not classified as zero-seat model-free" >&2 + exit 1 + } + +log="$(mktemp)" +catalog="" +cleanup() { + rm -f -- "$log" + if [ -n "$catalog" ]; then + case "$catalog" in + /tmp/st2e-[0-9]*) rm -rf -- "$catalog" ;; + *) echo "REFUSE: unexpected scratch catalog path $catalog" >&2 ;; + esac + fi +} +trap cleanup EXIT + +set +e +st2 eval ./cells/canonical-agent-runtime-smoke/ --keep >"$log" 2>&1 & +eval_pid=$! +catalog="/tmp/st2e-$eval_pid" +wait "$eval_pid" +status=$? +set -e +cat "$log" + +[ "$status" -eq 0 ] || { + echo "FAIL: canonical Agent Spec smoke exited $status" >&2 + exit "$status" +} +grep -Fq "SCORE: 9 PASS / 0 FAIL / 9 gating judges" "$log" +grep -Fq "VERDICT: PASS" "$log" + +for pid_file in "$catalog/probe/work.pid" "$catalog/probe/guard.pid"; do + pid="$(cat "$pid_file")" + [[ "$pid" =~ ^[0-9]+$ ]] || { + echo "FAIL: canonical runtime task receipt is not a PID: $pid_file=$pid" >&2 + exit 1 + } + if kill -0 "$pid" 2>/dev/null; then + echo "FAIL: canonical runtime task $pid from $pid_file survived eval teardown" >&2 + exit 1 + fi +done + +sessions="$(st2 pty --catalog "$catalog" ls --json)" +jq -e 'length == 0' <<<"$sessions" >/dev/null || { + echo "FAIL: canonical eval left PTY registry residue: $sessions" >&2 + exit 1 +} +test ! -e "$catalog/REMOTE-SPAWNED" + +echo "PASS: path-independent local PTY+exec launched, native routing completed, remote sentinel stayed inert, and teardown left zero tasks" diff --git a/bin/check-st2-pin-consistency.sh b/bin/check-st2-pin-consistency.sh index 6a276983..e4258acf 100755 --- a/bin/check-st2-pin-consistency.sh +++ b/bin/check-st2-pin-consistency.sh @@ -11,7 +11,8 @@ grep -Fq "$ST2_BINARY_SHA256" AGENT-SPEC.md grep -Fq "$ST2_BINARY_SHA256" README.md grep -Fq 'source bin/st2-pin.sh' bin/check-corpus.sh -if rg -n '9887b28|d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d' \ +if rg -n \ + '9887b28|d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d|0ec6a22|df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db' \ AGENT-SPEC.md README.md bin/check-corpus.sh bin/st2-pin.sh; then echo "FAIL: active st2 pin surfaces retain the superseded source or binary hash" >&2 exit 1 diff --git a/bin/st2-pin.sh b/bin/st2-pin.sh index 1784a70f..f784ad21 100755 --- a/bin/st2-pin.sh +++ b/bin/st2-pin.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -ST2_SOURCE_SHORT="0ec6a22" -ST2_SOURCE_FULL="0ec6a22e877f9374260202c5f76dabc4579fc7e3" -ST2_BINARY_SHA256="df0783843e5bb5b2bfd58323467b5ae0d89aebec4e12e5666139915e235ad2db" -ST2_VERSION_REGEX='^st2 0\.1\.0 — running from local source \(0ec6a22, .+ ago\)$' +ST2_SOURCE_SHORT="c6846f6" +ST2_SOURCE_FULL="c6846f6239329f0803142afc06c15a07b93937c1" +ST2_BINARY_SHA256="2bba8d58be24250bc262f75f835ce2d780369add275774f3f2135c623d23d29c" +ST2_VERSION_REGEX='^st2 0\.1\.0 — running from local source \(c6846f6, .+ ago\)$' diff --git a/cells/canonical-agent-runtime-smoke/README.md b/cells/canonical-agent-runtime-smoke/README.md index 9ba5201b..ebad02f9 100644 --- a/cells/canonical-agent-runtime-smoke/README.md +++ b/cells/canonical-agent-runtime-smoke/README.md @@ -1,20 +1,23 @@ # canonical-agent-runtime-smoke This model-free cell exercises st2's production Agent Spec eval seam with one -deterministic shell agent. The canonical declaration is the sole launch and -routing authority: st2 boots it, delivers the kickoff to its native inbox, -requires a fresh reply, grades the root and routing receipts, and tears the PTY -down through normal eval cleanup. +deterministic local shell agent and one valid remote-host sentinel. The local +declaration intentionally lives outside the conventional agent path and owns a +custom-named PTY plus an exec task. Its explicit identity and host remain the +sole launch and routing authority while the remote declaration stays inert. +st2 boots every local task, delivers the kickoff to the declaration's native +inbox, requires a fresh reply, captures the custom PTY log, and tears both +tasks down through normal eval cleanup. Before admission, a deterministic run creates the same disposable Git-worktree shape required by the paid interviewer. Strict materialization copies the canonical persona and bus overlays and applies `git-exclude` with zero warnings. It uses no Axe, account, model, or provider harness. -Its six gating results are the successful preparation run step, canonical team -completion, and four declared judges. The run-step result is emitted and graded -by st2 in addition to the judges declared in KDL. +Its nine gating results are the successful preparation run step, canonical +team completion, and seven declared judges. The run-step result is emitted and +graded by st2 in addition to the judges declared in KDL. ```sh -st2 eval ./cells/canonical-agent-runtime-smoke/ +bin/check-canonical-agent-runtime-smoke.sh ``` diff --git a/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl b/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl index 809b7f97..acfb7ff1 100644 --- a/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl +++ b/cells/canonical-agent-runtime-smoke/canonical-agent-runtime-smoke.kdl @@ -23,8 +23,17 @@ eval { judge "STRICT MATERIALIZATION — canonical persona and bus overlays materialize into a clean Git worktree" { exec #"test -s "$CATALOG/interviewer/.st2/PERSONA.md" && test -s "$CATALOG/interviewer/.st2/bus.md" && grep -Fxq ".st2/" "$CATALOG/interviewer/.git/info/exclude" && test -z "$(git -C "$CATALOG/interviewer" status --porcelain=v1)""# } - judge "MODEL FREE — the admitted Agent Spec contains no provider, account, or Axe launch" { - exec #"! grep -Eq 'axe|claude|codex|opencode|pi|--account' "$CATALOG/agents/evalhost/probe/agent.kdl""# + judge "PATH-INDEPENDENT ROUTING — the explicit local identity owns its declaration-parent inbox" { + exec #"test -d "$CATALOG/organization/.managed/arbitrary/declaration/resources/inbox""# + } + judge "GENERIC TASK VECTOR — custom PTY and exec task IDs were both admitted and launched" { + exec #"test -s "$CATALOG/logs/canonical-probe-work.log" && test -s "$CATALOG/probe/guard.pid" && kill -0 "$(cat "$CATALOG/probe/guard.pid")""# + } + judge "REMOTE PROJECTION — a valid remote-host declaration remains inert" { + exec #"test ! -e "$CATALOG/REMOTE-SPAWNED""# + } + judge "MODEL FREE — neither admitted declaration contains a provider, account, or Axe launch" { + exec #"! grep -ERq '(^|[^[:alnum:]_-])(axe|claude|codex|opencode|pi)([^[:alnum:]_-]|$)|--account' "$CATALOG/organization" "$CATALOG/fleet""# } } } diff --git a/cells/canonical-agent-runtime-smoke/fixture/fleet/remote/declaration/agent.kdl b/cells/canonical-agent-runtime-smoke/fixture/fleet/remote/declaration/agent.kdl new file mode 100644 index 00000000..fae9a85a --- /dev/null +++ b/cells/canonical-agent-runtime-smoke/fixture/fleet/remote/declaration/agent.kdl @@ -0,0 +1,8 @@ +agent "remote" { + identity "remote" + host "remotehost" + pty "work" { + id "canonical-remote-work" + command #"touch "$CATALOG/REMOTE-SPAWNED"; exec sleep 60"# + } +} diff --git a/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl b/cells/canonical-agent-runtime-smoke/fixture/organization/.managed/arbitrary/declaration/agent.kdl similarity index 52% rename from cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl rename to cells/canonical-agent-runtime-smoke/fixture/organization/.managed/arbitrary/declaration/agent.kdl index 426428c7..a8a388c6 100644 --- a/cells/canonical-agent-runtime-smoke/fixture/agents/evalhost/probe/agent.kdl +++ b/cells/canonical-agent-runtime-smoke/fixture/organization/.managed/arbitrary/declaration/agent.kdl @@ -5,7 +5,16 @@ agent "probe" { env { ST_AGENT "evalhost.probe" } - argv "bash" "$CATALOG/probe.sh" + + pty "work" { + id "canonical-probe-work" + argv "bash" "$CATALOG/probe.sh" + } + exec "guard" { + id "canonical-probe-guard" + command #"mkdir -p "$CATALOG/probe"; printf '%s\n' "$$$$" > "$CATALOG/probe/guard.pid"; exec sleep 60"# + } + render { copy "_templates/generalist.md" ".st2/PERSONA.md" copy "_templates/bus.st2.md" ".st2/bus.md" diff --git a/cells/canonical-agent-runtime-smoke/fixture/probe.sh b/cells/canonical-agent-runtime-smoke/fixture/probe.sh index 6ef660a6..26a31ae0 100755 --- a/cells/canonical-agent-runtime-smoke/fixture/probe.sh +++ b/cells/canonical-agent-runtime-smoke/fixture/probe.sh @@ -5,6 +5,8 @@ test "$CATALOG" = "$ST_ROOT" test "$PTY_ROOT" = "$CATALOG/pty" test "$ST_AGENT" = "evalhost.probe" mkdir -p "$CATALOG/probe" +printf '%s\n' "$$" >"$CATALOG/probe/work.pid" +echo "canonical probe ready" touch "$CATALOG/probe/roots-ok" for _ in $(seq 1 200); do From 9abb46efcb52f1c3ea05b95c2f0566fbeac6ae46 Mon Sep 17 00:00:00 2001 From: Nathan Date: Thu, 30 Jul 2026 22:56:00 +0200 Subject: [PATCH 09/12] Use agent terminology in PR56 gates --- AGENT-SPEC.md | 2 +- CATALOG.md | 38 +++++++++---------- README.md | 2 +- bin/check-canonical-agent-runtime-smoke.sh | 2 +- ...eck-canonical-agent-template-mutations.sh} | 2 +- ...e.sh => check-canonical-agent-template.sh} | 1 + bin/check-corpus.sh | 6 +-- bin/check-event-first.sh | 6 +-- bin/check-harness-contract.sh | 8 ++-- bin/check-preflight-safety.sh | 10 ++--- bin/corpus-inventory.sh | 22 +++++------ bin/generate-catalog.sh | 12 +++--- ...-inventory.sh => model-agent-inventory.sh} | 8 ++-- bin/overnight.sh | 18 ++++----- docs/vrs/spec.md | 10 ++--- evidence/exclusions.tsv | 2 +- evidence/harness-exclusions.tsv | 30 +++++++-------- 17 files changed, 90 insertions(+), 89 deletions(-) rename bin/{check-canonical-seat-template-mutations.sh => check-canonical-agent-template-mutations.sh} (95%) rename bin/{check-canonical-seat-template.sh => check-canonical-agent-template.sh} (95%) rename bin/{model-seat-inventory.sh => model-agent-inventory.sh} (93%) diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 6f38b50f..803c6e97 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -421,7 +421,7 @@ example, create every `$CATALOG`-rooted workspace it names. ## Free authoring gate -These commands parse and materialize without starting a model seat: +These commands parse and materialize without starting a model agent: ```sh st2 validate --catalog "$CATALOG" --host --strict diff --git a/CATALOG.md b/CATALOG.md index 916011c0..0bba34d3 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -9,12 +9,12 @@ structured run row say so explicitly. Every model launch is explicitly pinned by `bin/check-model-policy.sh`: Claude uses `claude-sonnet-5` at medium effort and Codex uses `gpt-5.6-sol` at medium reasoning effort. -Every bus-connected model seat is mechanically checked for the event-first DING lifecycle and its +Every bus-connected model agent is mechanically checked for the event-first DING lifecycle and its harness-native loader plus canonical hook file. ## Included overnight inventory -| Cell | Harness | Model(s) / effort | Model seats | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | +| Cell | Harness | Model(s) / effort | Model agents | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | |---|---|---|---:|---|---|---:|---|---| | `agent-new-bundle-contract` | model-free | — | 0 | none | `60s` | 5 | — | **NO STRUCTURED RUN** | | `agent-new-interview` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | @@ -77,30 +77,30 @@ while the last-run column makes a recorded failure distinct from a cell with no - `clean-compose` — retired composition-overlay discriminator - `compose-config-load` — retired composition/config-loader discriminator - `compose-global-skill` — retired composition/global-skill discriminator -- `team-standup` — retired runtime seat-generation discriminator +- `team-standup` — retired runtime agent-generation discriminator `hook-integrity` is not excluded: it is now a model-free, current-native `st2 up --materialize-only` hook installation and idempotency test. ## Harness hook exclusions -- `agent-new-bundle-contract` / `cell` — deterministic semantic-intent lowering and mutation probe; no harness seat -- `canonical-agent-runtime-smoke` / `cell` — deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness seat -- `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness seat +- `agent-new-bundle-contract` / `cell` — deterministic semantic-intent lowering and mutation probe; no harness agent +- `canonical-agent-runtime-smoke` / `cell` — deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness agent +- `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness agent - `docs` / `judge:cold-reader` — one-shot offline Claude print grader; no bus identity, DING, or hook surface -- `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness seat -- `hook-integrity` / `cell` — deterministic native hook-installation probe; no live harness seat -- `host-lock-health-negatives` / `cell` — deterministic native doctor negative probe; no harness seat -- `pty-attach-only` / `cell` — deterministic PTY dead-attach policy probe; no harness seat -- `pty-send-peek` / `cell` — deterministic PTY transport probe; no harness seat -- `presence-ding-matrix` / `cell` — deterministic native presence and DING probe; no harness seat -- `reconcile-retire-keep` / `cell` — deterministic native reconcile lifecycle probe; no harness seat -- `render-target-safety` / `cell` — deterministic native render-policy probe; no harness seat -- `strict-validation-json` / `cell` — deterministic native validation-diagnostics probe; no harness seat -- `st2-doctor-structure` / `cell` — deterministic catalog-health probe; no harness seat -- `st2-network` / `cell` — deterministic catalog/message round-trip probe; no harness seat +- `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness agent +- `hook-integrity` / `cell` — deterministic native hook-installation probe; no live harness agent +- `host-lock-health-negatives` / `cell` — deterministic native doctor negative probe; no harness agent +- `pty-attach-only` / `cell` — deterministic PTY dead-attach policy probe; no harness agent +- `pty-send-peek` / `cell` — deterministic PTY transport probe; no harness agent +- `presence-ding-matrix` / `cell` — deterministic native presence and DING probe; no harness agent +- `reconcile-retire-keep` / `cell` — deterministic native reconcile lifecycle probe; no harness agent +- `render-target-safety` / `cell` — deterministic native render-policy probe; no harness agent +- `strict-validation-json` / `cell` — deterministic native validation-diagnostics probe; no harness agent +- `st2-doctor-structure` / `cell` — deterministic catalog-health probe; no harness agent +- `st2-network` / `cell` — deterministic catalog/message round-trip probe; no harness agent - `targeted-reconcile-isolation` / `cell` — deterministic selected reconcile isolation probe; no bus-connected agent -- `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness seat +- `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness agent These **17** rows are generated from structured exclusions. A cell-level row is required for every derived model-free cell. The one-shot docs cold reader is a model-backed grader, but not a bus-connected @@ -114,7 +114,7 @@ Run the complete model-free preflight: bin/check-corpus.sh ``` -Preview the exact lexical run order, harness, model, effort, seat count, cost band, and timeout: +Preview the exact lexical run order, harness, model, effort, agent count, cost band, and timeout: ```sh bin/overnight.sh --dry-run diff --git a/README.md b/README.md index 1e84f970..f8512ebf 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ next paid cell to start. ## Run one cell `st2 eval` creates a hermetic temporary catalog, copies the fixture, boots declared agents and model judges, -delivers the kickoff, waits for completion or the cell timeout, tears seats down, and runs the held-out judges: +delivers the kickoff, waits for completion or the cell timeout, tears agents down, and runs the held-out judges: ```sh st2 eval ./cells/ghost-bug/ diff --git a/bin/check-canonical-agent-runtime-smoke.sh b/bin/check-canonical-agent-runtime-smoke.sh index af6ec31f..74725011 100755 --- a/bin/check-canonical-agent-runtime-smoke.sh +++ b/bin/check-canonical-agent-runtime-smoke.sh @@ -19,7 +19,7 @@ sha256="$(sha256sum "$st2_bin" | awk '{ print $1 }')" bin/corpus-inventory.sh --no-header | awk -F '\t' '$1 == "canonical-agent-runtime-smoke" && $2 == "model-free" && $5 == 0 { found = 1 } END { exit !found }' || { - echo "FAIL: canonical-agent-runtime-smoke is not classified as zero-seat model-free" >&2 + echo "FAIL: canonical-agent-runtime-smoke is not classified as zero-model-agent model-free" >&2 exit 1 } diff --git a/bin/check-canonical-seat-template-mutations.sh b/bin/check-canonical-agent-template-mutations.sh similarity index 95% rename from bin/check-canonical-seat-template-mutations.sh rename to bin/check-canonical-agent-template-mutations.sh index 8aecf918..438755e2 100755 --- a/bin/check-canonical-seat-template-mutations.sh +++ b/bin/check-canonical-agent-template-mutations.sh @@ -2,7 +2,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -checker="$repo_root/bin/check-canonical-seat-template.sh" +checker="$repo_root/bin/check-canonical-agent-template.sh" source_template="$repo_root/cells/agent-new-interview/fixture/agent.kdl.template" source_publisher="$repo_root/cells/agent-new-interview/fixture/publish-interviewer.sh" scratch="$(mktemp -d)" diff --git a/bin/check-canonical-seat-template.sh b/bin/check-canonical-agent-template.sh similarity index 95% rename from bin/check-canonical-seat-template.sh rename to bin/check-canonical-agent-template.sh index 418c7119..cb3d64de 100755 --- a/bin/check-canonical-seat-template.sh +++ b/bin/check-canonical-agent-template.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Validate the canonical paid model-agent declaration template and publisher. set -euo pipefail template="${1:?canonical Agent Spec template required}" diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index 5af09ab3..7e8063be 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Complete free preflight for the current overnight corpus. Never starts a model seat. +# Complete free preflight for the current overnight corpus. Never starts a model agent. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -30,7 +30,7 @@ bin/check-preflight-safety.sh bin/check-agent-new-renderer-security.sh bin/check-model-policy.sh bin/check-model-policy-mutations.sh -bin/check-canonical-seat-template-mutations.sh +bin/check-canonical-agent-template-mutations.sh bin/check-run-history.sh attempt_baseline="${EVALS_EVIDENCE_BASELINE_REF:-}" if [ -z "$attempt_baseline" ] && git rev-parse --verify origin/main^{commit} >/dev/null 2>&1; then @@ -48,7 +48,7 @@ else fi bin/check-agent-new-interview-attempts-mutations.sh bin/check-retired-surfaces.sh -bin/model-seat-inventory.sh >/dev/null +bin/model-agent-inventory.sh >/dev/null bin/check-event-first.sh bin/check-kdl-parse.sh bin/check-st2-semantic.sh diff --git a/bin/check-event-first.sh b/bin/check-event-first.sh index ef449c29..cd681b78 100755 --- a/bin/check-event-first.sh +++ b/bin/check-event-first.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Enforce one cold-start drain followed by native DING wakeups for every maintained model seat. +# Enforce one cold-start drain followed by native DING wakeups for every maintained model agent. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -12,7 +12,7 @@ cleanup() { } trap cleanup EXIT -bin/model-seat-inventory.sh --no-header > "$inventory" +bin/model-agent-inventory.sh --no-header > "$inventory" failed=0 fail() { @@ -69,7 +69,7 @@ if rg -n --pcre2 \ fi if [ "$failed" -eq 0 ]; then - printf 'PASS: %s model seats use one cold-start drain and event-first native DING wakeups\n' \ + printf 'PASS: %s model agents use one cold-start drain and event-first native DING wakeups\n' \ "$(wc -l < "$inventory" | tr -d ' ')" fi exit "$failed" diff --git a/bin/check-harness-contract.sh b/bin/check-harness-contract.sh index af061b96..e46765f0 100755 --- a/bin/check-harness-contract.sh +++ b/bin/check-harness-contract.sh @@ -30,7 +30,7 @@ cleanup() { } trap cleanup EXIT -bin/model-seat-inventory.sh --no-header > "$inventory" +bin/model-agent-inventory.sh --no-header > "$inventory" hydrate_gitdirs() { local root="$1" gitdir target @@ -124,7 +124,7 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent source_kind source axe_launch=0 if [ "$source_kind" = "canonical-template" ]; then publisher="$(dirname "$source_path")/publish-interviewer.sh" - bin/check-canonical-seat-template.sh "$source_path" "$publisher" >/dev/null || { + bin/check-canonical-agent-template.sh "$source_path" "$publisher" >/dev/null || { failed=1 continue } @@ -182,7 +182,7 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent source_kind source done < "$inventory" [ "$checked" -gt 0 ] || { - echo "FAIL: no $selected model seats were checked" >&2 + echo "FAIL: no $selected model agents were checked" >&2 exit 1 } @@ -209,7 +209,7 @@ grep -Fxq $'docs\tjudge:cold-reader\tone-shot offline Claude print grader; no bu } if [ "$failed" -eq 0 ]; then - printf 'PASS: %d %s model seats materialize and use the canonical harness overlay and hooks\n' \ + printf 'PASS: %d %s model agents materialize and use the canonical harness overlay and hooks\n' \ "$checked" "$selected" printf 'PASS: %d derived model-free cells have explicit harness-hook exclusions\n' \ "${#expected[@]}" diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index 1f71275e..bd8e21ce 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -11,8 +11,8 @@ reachable=( bin/check-agent-new-interview-attempts-mutations.sh bin/check-agent-new-renderer-security.sh bin/check-agent-new-interview-attempts.sh - bin/check-canonical-seat-template-mutations.sh - bin/check-canonical-seat-template.sh + bin/check-canonical-agent-template-mutations.sh + bin/check-canonical-agent-template.sh bin/check-event-first.sh bin/check-fixture-reset.sh bin/check-fixture-reset-terminal.sh @@ -33,7 +33,7 @@ reachable=( bin/check-weird-git-setup.sh bin/corpus-inventory.sh bin/generate-catalog.sh - bin/model-seat-inventory.sh + bin/model-agent-inventory.sh bin/st2-pin.sh ) materializers=( @@ -71,7 +71,7 @@ expected_direct=( bin/check-agent-new-interview-attempts-mutations.sh bin/check-agent-new-interview-attempts.sh bin/check-agent-new-renderer-security.sh - bin/check-canonical-seat-template-mutations.sh + bin/check-canonical-agent-template-mutations.sh bin/check-event-first.sh bin/check-fixture-reset-terminal.sh bin/check-harness-contract.sh @@ -90,7 +90,7 @@ expected_direct=( bin/check-vrs-variations.sh bin/check-weird-git-setup.sh bin/generate-catalog.sh - bin/model-seat-inventory.sh + bin/model-agent-inventory.sh ) if [ "${direct[*]}" != "${expected_direct[*]}" ]; then fail "check-corpus.sh direct command set differs from the reviewed allowlist" diff --git a/bin/corpus-inventory.sh b/bin/corpus-inventory.sh index 7fdd67b9..36979f76 100755 --- a/bin/corpus-inventory.sh +++ b/bin/corpus-inventory.sh @@ -13,15 +13,15 @@ elif [ "$#" -ne 0 ]; then exit 2 fi -seat_inventory="$(mktemp)" +agent_inventory="$(mktemp)" cleanup() { - rm -f -- "$seat_inventory" + rm -f -- "$agent_inventory" } trap cleanup EXIT -bin/model-seat-inventory.sh --no-header > "$seat_inventory" +bin/model-agent-inventory.sh --no-header > "$agent_inventory" if [ "$include_header" -eq 1 ]; then - printf 'cell\tharness\tmodels\teffort\tmodel_seats\tcost_band\ttimeout\theld_out_judges\n' + printf 'cell\tharness\tmodels\teffort\tmodel_agents\tcost_band\ttimeout\theld_out_judges\n' fi mapfile -t cells < <(find cells -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | LC_ALL=C sort) @@ -52,10 +52,10 @@ for cell in "${cells[@]}"; do } timeout="${timeouts[0]}" - claude="$(awk -F '\t' -v cell="$cell" '$1 == cell && $3 == "Claude" { count += 1 } END { print count + 0 }' "$seat_inventory")" - codex="$(awk -F '\t' -v cell="$cell" '$1 == cell && $3 == "Codex" { count += 1 } END { print count + 0 }' "$seat_inventory")" + claude="$(awk -F '\t' -v cell="$cell" '$1 == cell && $3 == "Claude" { count += 1 } END { print count + 0 }' "$agent_inventory")" + codex="$(awk -F '\t' -v cell="$cell" '$1 == cell && $3 == "Codex" { count += 1 } END { print count + 0 }' "$agent_inventory")" - seats=$((claude + codex)) + agents=$((claude + codex)) if [ "$claude" -gt 0 ] && [ "$codex" -gt 0 ]; then harness="mixed" models="claude-sonnet-5+gpt-5.6-sol" @@ -70,13 +70,13 @@ for cell in "${cells[@]}"; do models="-" fi - if [ "$seats" -eq 0 ]; then + if [ "$agents" -eq 0 ]; then effort="-" cost="none" - elif [ "$seats" -eq 1 ]; then + elif [ "$agents" -eq 1 ]; then effort="medium" cost="low" - elif [ "$seats" -eq 2 ]; then + elif [ "$agents" -eq 2 ]; then effort="medium" cost="medium" else @@ -87,5 +87,5 @@ for cell in "${cells[@]}"; do judges="$(rg -c '^[[:space:]]*judge[[:space:]]+"' "$kdl" || true)" judges="${judges:-0}" printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "$cell" "$harness" "$models" "$effort" "$seats" "$cost" "$timeout" "$judges" + "$cell" "$harness" "$models" "$effort" "$agents" "$cost" "$timeout" "$judges" done diff --git a/bin/generate-catalog.sh b/bin/generate-catalog.sh index 0a70e45e..2ba81300 100755 --- a/bin/generate-catalog.sh +++ b/bin/generate-catalog.sh @@ -141,7 +141,7 @@ done < evidence/harness-exclusions.tsv total=0 model_free=0 model_backed=0 -while IFS=$'\t' read -r cell harness models effort seats cost timeout judges; do +while IFS=$'\t' read -r cell harness models effort agents cost timeout judges; do ((total += 1)) if [ "$harness" = "model-free" ]; then ((model_free += 1)) @@ -163,16 +163,16 @@ structured run row say so explicitly. Every model launch is explicitly pinned by \`bin/check-model-policy.sh\`: Claude uses \`claude-sonnet-5\` at medium effort and Codex uses \`gpt-5.6-sol\` at medium reasoning effort. -Every bus-connected model seat is mechanically checked for the event-first DING lifecycle and its +Every bus-connected model agent is mechanically checked for the event-first DING lifecycle and its harness-native loader plus canonical hook file. ## Included overnight inventory -| Cell | Harness | Model(s) / effort | Model seats | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | +| Cell | Harness | Model(s) / effort | Model agents | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | |---|---|---|---:|---|---|---:|---|---| EOF - while IFS=$'\t' read -r cell harness models effort seats cost timeout judges; do + while IFS=$'\t' read -r cell harness models effort agents cost timeout judges; do if [ "$models" = "-" ]; then model_display="—" else @@ -203,7 +203,7 @@ EOF last="**NO STRUCTURED RUN**" fi printf '| `%s` | %s | %s | %s | %s | `%s` | %s | %s | %s |\n' \ - "$cell" "$harness" "$model_display" "$seats" "$cost" "$timeout" "$judges" "$accepted" "$last" + "$cell" "$harness" "$model_display" "$agents" "$cost" "$timeout" "$judges" "$accepted" "$last" done < "$inventory" cat <&2 + echo "usage: bin/model-agent-inventory.sh [--no-header]" >&2 exit 2 fi @@ -142,7 +142,7 @@ while IFS= read -r template; do fi if [ -z "$agent" ] || [ -z "$workspace" ] || [ -z "$st_agent" ] || [ -z "$command_line" ] || [ -z "$harness" ] || [ "$ding_count" -ne 1 ]; then - echo "FAIL: canonical model-seat template is incomplete: $template" >&2 + echo "FAIL: canonical model-agent template is incomplete: $template" >&2 failed=1 continue fi @@ -161,7 +161,7 @@ launches="$( )" rows="$(wc -l < "$inventory" | tr -d ' ')" if [ "$rows" -ne "$launches" ]; then - echo "FAIL: model-seat inventory found $rows structured seats but $launches root-KDL launch lines" >&2 + echo "FAIL: model-agent inventory found $rows structured agents but $launches root-KDL launch lines" >&2 failed=1 fi diff --git a/bin/overnight.sh b/bin/overnight.sh index 5640810f..a68630e0 100755 --- a/bin/overnight.sh +++ b/bin/overnight.sh @@ -128,16 +128,16 @@ fi requires_claude=0 requires_codex=0 -while IFS=$'\t' read -r _cell _harness models _effort _seats _cost _timeout _judges; do +while IFS=$'\t' read -r _cell _harness models _effort _agents _cost _timeout _judges; do [[ "$models" != *claude-sonnet-5* ]] || requires_claude=1 [[ "$models" != *gpt-5.6-sol* ]] || requires_codex=1 done < "$inventory" printf '%-30s %-10s %-35s %-7s %-5s %-8s %s\n' \ - CELL HARNESS MODEL EFFORT SEATS COST TIMEOUT -while IFS=$'\t' read -r cell harness models effort seats cost timeout _judges; do + CELL HARNESS MODEL EFFORT AGENTS COST TIMEOUT +while IFS=$'\t' read -r cell harness models effort agents cost timeout _judges; do printf '%-30s %-10s %-35s %-7s %-5s %-8s %s\n' \ - "$cell" "$harness" "$models" "$effort" "$seats" "$cost" "$timeout" + "$cell" "$harness" "$models" "$effort" "$agents" "$cost" "$timeout" done < "$inventory" cells="$(wc -l < "$inventory" | tr -d ' ')" @@ -179,7 +179,7 @@ fi } echo -echo "== free preflight (no model seats) ==" +echo "== free preflight (no model agents) ==" bin/check-corpus.sh if [ "$requires_claude" -eq 1 ]; then @@ -281,7 +281,7 @@ cleanup_timed_out_catalog() { fi } -while IFS=$'\t' read -r cell harness models effort seats cost declared_timeout _judges; do +while IFS=$'\t' read -r cell harness models effort agents cost declared_timeout _judges; do hash="$(cell_hash "$cell")" receipt="$state_dir/receipts/$cell.env" if [ -f "$receipt" ] && @@ -295,8 +295,8 @@ while IFS=$'\t' read -r cell harness models effort seats cost declared_timeout _ stamp="$(date -u +%Y%m%dT%H%M%SZ)" log="$state_dir/logs/$cell.$stamp.log" watchdog_seconds=$(($(duration_seconds "$declared_timeout") + 180)) - printf '\n== %s: %s, %s, %s seat(s), %s cost, timeout %s (+180s watchdog) ==\n' \ - "$cell" "$harness" "$models" "$seats" "$cost" "$declared_timeout" + printf '\n== %s: %s, %s, %s agent(s), %s cost, timeout %s (+180s watchdog) ==\n' \ + "$cell" "$harness" "$models" "$agents" "$cost" "$declared_timeout" setsid stdbuf -oL -eL st2 eval "./cells/$cell/" --keep > "$log" 2>&1 & eval_pid=$! @@ -404,7 +404,7 @@ while IFS=$'\t' read -r cell harness models effort seats cost declared_timeout _ "harness=$harness" \ "models=$models" \ "effort=$effort" \ - "model_seats=$seats" \ + "model_agents=$agents" \ "cost_band=$cost" \ "declared_timeout=$declared_timeout" \ "hard_usage_warning=$hard_usage_seen" \ diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index 180d4607..488c884a 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -93,9 +93,9 @@ one-sentence human request evaluates the renderer contract across multiple intents and closed-input failures. Held-out mutation controls reject raw provider argv, drift between intent and KDL, and missing initial context. -- **R06-R09:** The judgment cell uses one explicitly pinned Claude Sonnet seat +- **R06-R09:** The judgment cell uses one explicitly pinned Claude Sonnet agent at medium effort and a five-minute cell timeout. The deterministic cell has - no model seat and a one-minute timeout. Normal st2 eval cleanup and receipt + no model agent and a one-minute timeout. Normal st2 eval cleanup and receipt policy applies to both. - The interviewer output is semantic intent, not KDL. Stable launch axes are explicit in that intent; the deterministic boundary owns canonical KDL, @@ -109,7 +109,7 @@ one-sentence human request `axe agent launch` with explicit harness, model, effort, persona, mode, and boot axes. The cell does not pin `--account`: Axe selects an eligible account per run. Corpus inventory, model policy, lifecycle, and harness - checks recognize this typed launch as a paid model seat and reject durable + checks recognize this typed launch as a paid model agent and reject durable account pins. - The paid cell uses st2's explicit `canonical-agents` seam. A deterministic pre-admission run first creates the copied interviewer workspace as a clean, @@ -118,12 +118,12 @@ one-sentence human request Axe adapter, absolute profile path, and canonical persona source. st2 then carries that declaration unchanged through strict validation, warning-free materialization, launch, kickoff routing, singleton completion, - and teardown. No compact eval seat, compatibility wrapper, account pin, or + and teardown. No compact eval agent, compatibility wrapper, account pin, or ambient provider launch participates. - The paid cell is environment-bound to that explicit runtime-profile artifact; the artifact identity belongs in run evidence and results are not represented as hermetic across different profiles. -- One structured seat inventory is authoritative for both compact eval seats +- One structured model-agent inventory is authoritative for both compact eval agents and canonical paid templates. Corpus cost classification, event-first policy, and harness-overlay checks consume its source-kind/path records; canonical template mutations prove required launch axes and overlays are diff --git a/evidence/exclusions.tsv b/evidence/exclusions.tsv index f513ae35..05614619 100644 --- a/evidence/exclusions.tsv +++ b/evidence/exclusions.tsv @@ -2,4 +2,4 @@ cell reason clean-compose retired composition-overlay discriminator compose-config-load retired composition/config-loader discriminator compose-global-skill retired composition/global-skill discriminator -team-standup retired runtime seat-generation discriminator +team-standup retired runtime agent-generation discriminator diff --git a/evidence/harness-exclusions.tsv b/evidence/harness-exclusions.tsv index f785a70e..acbf8d0e 100644 --- a/evidence/harness-exclusions.tsv +++ b/evidence/harness-exclusions.tsv @@ -1,18 +1,18 @@ cell surface reason -agent-new-bundle-contract cell deterministic semantic-intent lowering and mutation probe; no harness seat -canonical-agent-runtime-smoke cell deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness seat -context-resource-continuity cell deterministic native context/resource restart probe; no harness seat +agent-new-bundle-contract cell deterministic semantic-intent lowering and mutation probe; no harness agent +canonical-agent-runtime-smoke cell deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness agent +context-resource-continuity cell deterministic native context/resource restart probe; no harness agent docs judge:cold-reader one-shot offline Claude print grader; no bus identity, DING, or hook surface -exec-lifecycle-logging cell deterministic native exec lifecycle probe; no harness seat -hook-integrity cell deterministic native hook-installation probe; no live harness seat -host-lock-health-negatives cell deterministic native doctor negative probe; no harness seat -pty-attach-only cell deterministic PTY dead-attach policy probe; no harness seat -pty-send-peek cell deterministic PTY transport probe; no harness seat -presence-ding-matrix cell deterministic native presence and DING probe; no harness seat -reconcile-retire-keep cell deterministic native reconcile lifecycle probe; no harness seat -render-target-safety cell deterministic native render-policy probe; no harness seat -strict-validation-json cell deterministic native validation-diagnostics probe; no harness seat -st2-doctor-structure cell deterministic catalog-health probe; no harness seat -st2-network cell deterministic catalog/message round-trip probe; no harness seat +exec-lifecycle-logging cell deterministic native exec lifecycle probe; no harness agent +hook-integrity cell deterministic native hook-installation probe; no live harness agent +host-lock-health-negatives cell deterministic native doctor negative probe; no harness agent +pty-attach-only cell deterministic PTY dead-attach policy probe; no harness agent +pty-send-peek cell deterministic PTY transport probe; no harness agent +presence-ding-matrix cell deterministic native presence and DING probe; no harness agent +reconcile-retire-keep cell deterministic native reconcile lifecycle probe; no harness agent +render-target-safety cell deterministic native render-policy probe; no harness agent +strict-validation-json cell deterministic native validation-diagnostics probe; no harness agent +st2-doctor-structure cell deterministic catalog-health probe; no harness agent +st2-network cell deterministic catalog/message round-trip probe; no harness agent targeted-reconcile-isolation cell deterministic selected reconcile isolation probe; no bus-connected agent -two-networks-coexist cell deterministic catalog-isolation probe; no harness seat +two-networks-coexist cell deterministic catalog-isolation probe; no harness agent From c40cb7643688a249f615c660fb134ddf19af0962 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:20:23 +0200 Subject: [PATCH 10/12] evals: pin canonical st2 package provenance agent-session-id: dev3.dotfiles-cos-misc-agent-runtime-simplification agent-tool: Codex agent-tool-version: 0.145.0 agent-model: gpt-5.6-sol agent-runtime-profile: /home/schickling/.config/coding-agents/profile.json agent-skills-manifest: /nix/store/nk9iml2841l1yjjg0f6f0d3y60zkg1nn-agent-skills-corpus/share/agent-skills/manifest.json tooling-profile: dotfiles@4a0515f --- AGENT-SPEC.md | 79 +++++++++++++------ README.md | 16 ++-- bin/check-canonical-agent-runtime-smoke.sh | 13 +-- bin/check-corpus.sh | 15 +--- bin/check-preflight-safety.sh | 5 +- bin/check-st2-package-provenance-mutations.sh | 34 ++++++++ bin/check-st2-package-provenance.sh | 75 ++++++++++++++++++ bin/check-st2-pin-consistency.sh | 14 +++- bin/st2-pin.sh | 11 ++- docs/vrs/spec.md | 11 +++ evidence/st2-package-provenance-20260730.md | 27 +++++++ 11 files changed, 242 insertions(+), 58 deletions(-) create mode 100755 bin/check-st2-package-provenance-mutations.sh create mode 100755 bin/check-st2-package-provenance.sh create mode 100644 evidence/st2-package-provenance-20260730.md diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index 803c6e97..de1f966e 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -45,6 +45,7 @@ agent "" { supervisor "." retired #false keep #false + lifecycle "service" restart { attempts 3 @@ -57,7 +58,7 @@ agent "" { ST_AGENT "." } - command #""# + argv "" "" ding render { @@ -82,9 +83,11 @@ Supported agent children are: | `supervisor "…"` | Optional bare identity or full bus id for crash-loop routing. | | `retired #true` | Decommission the declaration on the next reconciliation. Edit this flag; do not delete a live declaration to retire it. | | `keep #true` | Freeze dead evidence and suppress collection/restart for every task; retirement still stops live tasks. | +| `lifecycle "service"|"adopt-only"` | Reconciliation policy for the compact task. Explicit tasks declare their own policy. | | `restart { … }` | Optional service restart policy. | | `env { KEY "value" }` | Environment inherited by the compact agent task and sidecars. | -| `command "…"` | Compact interactive task named `agent`. | +| `argv "program" "arg"…` | Preferred compact interactive task named `agent`; direct program invocation with preserved argument boundaries. | +| `command "…"` | Compact interactive shell task named `agent`; use only when shell syntax is intentional. | | `ding` | Compact native DING sidecar named `ding`. | | `pty "name" { … }` | Explicit interactive task. | | `exec "name" { … }` | Explicit non-interactive task. | @@ -102,20 +105,22 @@ rely on that permissiveness. ## Compact and explicit tasks -The canonical compact pair: +The canonical compact pair uses direct argv: ```kdl -command #""# +argv "" "" ding ``` lowers to an interactive `pty "agent"` and a non-interactive `exec "ding"` sidecar. Do not declare -both `command` and `pty "agent"`, or both `ding` and `exec "ding"`. +both compact `argv` and compact `command`; do not combine either compact launch with `pty "agent"`; +and do not declare both `ding` and `exec "ding"`. The compact task has id equal to the agent bus id, tag `role=agent`, and inherited agent environment. The derived DING task has id `.ding` and inherited environment. A service is runnable only when at least -one authored task has a command; the derived sidecar alone is insufficient, including on a retired -declaration. Task names are sorted lexically after lowering. +one authored task has exactly one `argv` or `command`; `argv` must have a non-empty program element. The +derived sidecar alone is insufficient, including on a retired declaration. Task names are sorted lexically +after lowering. Use explicit tasks only when the agent needs an additional managed process or task-specific configuration: @@ -123,9 +128,10 @@ configuration: ```kdl pty "agent" { id "." - command #""# + argv "" "" cwd "/absolute/workspace/or/$CATALOG/path" keep #false + lifecycle "service" tags role="agent" purpose="subject" env { ST_AGENT "." @@ -137,6 +143,7 @@ exec "helper" { command #"exec ./long-running-helper"# cwd "/absolute/workspace/or/$CATALOG/path" keep #false + lifecycle "adopt-only" tags purpose="fixture" env { EXAMPLE "value" @@ -144,11 +151,17 @@ exec "helper" { } ``` -Each explicit task supports only `id`, `command`, `cwd`, `keep`, `tags`, and `env`. A nameless task is a -validation error. Agent-level environment is parsed before tasks and inherited independent of declaration -order; task-level values override it. A missing `cwd` falls back to agent workspace, then the spec directory. -Commands run through `sh -c`. Interactive harnesses belong in `pty`; terminal-free daemons and helpers belong -in `exec`. +Each explicit task supports only `id`, `argv`, `command`, `cwd`, `keep`, `lifecycle`, `tags`, and `env`. +A nameless task, empty `argv`, or task carrying both `argv` and `command` is a validation error. Agent-level +environment is parsed before tasks and inherited independent of declaration order; task-level values override +it. A missing `cwd` falls back to agent workspace, then the spec directory. + +`argv` resolves a bare element zero through the task environment's `PATH`, expands st2 environment references +in each element, preserves argument boundaries, and launches the program directly without a shell. `command` +is opaque shell source run verbatim under `sh -c`; use it only for intentional shell behavior such as pipes, +redirections, compound commands, or shell expansion. Writing `exec` inside `command` can remove the inner +shell after it interprets that source, but it does not turn the declaration into direct argv. Interactive +harnesses belong in `pty`; terminal-free daemons and helpers belong in `exec`. Interactive tasks run detached through the PTY runtime with stable lifecycle ids. Exec tasks allocate no terminal, run in their own detached process group, append diagnostics below `/logs/`, and retain one @@ -159,7 +172,8 @@ what stops them. ## Environment and expansion Expansion recognizes `$VAR`, `${VAR}`, and `$$` for a literal dollar. Unset variables remain literal. Task -environment values, tags, and cwd expand before spawn; the command remains opaque for its `sh -c`. +environment values, tags, cwd, and each direct argv element expand before spawn; command remains opaque for +its `sh -c`. st2 defaults `CATALOG` and `ST_ROOT` to the catalog. Effective `PTY_ROOT` is a non-empty ambient value or `/pty`; authored task environment cannot override it. st2 supplies `ST_HOOKS` only when an installed @@ -225,13 +239,21 @@ misclassified as an empty registry. Reconciliation is per task: - active and alive: adopt; -- missing: launch only that task; -- dead non-keep: preserve bounded diagnostics, collect, and restart according to policy; +- missing service task: launch only that task; +- dead non-keep service task: preserve bounded diagnostics, collect, and restart according to policy; - dead keep: freeze evidence without collection or restart; +- missing or dead `adopt-only` task: report `held` without collection, replacement, or launch; - retired and live: stop even when keep is set; - retired and dead non-keep: final collection; - unrendered or unrunnable: do not launch. +`lifecycle "adopt-only"` is a migration fence, not a restart policy. st2 may adopt an already-live generation +but may not create a missing generation or reap and replace a dead one. Declare it at agent level for the +compact task or inside each applicable explicit task. Explicit tasks that omit it independently default to +`service`; they do not inherit the compact-task policy. Deliberately restore `service` (or remove the field) +to authorize normal launch/replacement. `retired #true` is the separate teardown authority and takes +precedence. + `role` has no branch in lifecycle behavior. `supervisor` is used for `ST_SUPERVISOR` and best-effort crash-loop routing. New exact Codex launches require verified installed hooks and workspace pretrust; already live/adopted Codex tasks are not stopped by that launch gate. @@ -267,7 +289,7 @@ agent "" { ST_AGENT "." } - command #"exec claude --model claude-sonnet-5 --effort medium --permission-mode bypassPermissions ''"# + argv "claude" "--model" "claude-sonnet-5" "--effort" "medium" "--permission-mode" "bypassPermissions" "" ding render { @@ -334,7 +356,7 @@ agent "" { ST_AGENT "." } - command #"exec codex --model gpt-5.6-sol -c 'model_reasoning_effort="medium"' --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust ''"# + argv "codex" "--model" "gpt-5.6-sol" "-c" "model_reasoning_effort=\"medium\"" "--dangerously-bypass-approvals-and-sandbox" "--dangerously-bypass-hook-trust" "" ding render { @@ -409,7 +431,7 @@ native DING wakeups. Structured exceptions are in `evidence/harness-exclusions.t At `agents/example/worker/agent.kdl`, path-derived identity and host make this the smallest valid service: ```kdl -agent { command "true" } +agent { argv "true" } ``` At the pinned source it validates as one agent with zero errors and warnings. Production declarations should @@ -432,8 +454,17 @@ Inspect the declaration, every referenced template, and every workspace destinat materialization command. Materialization is byte-idempotent and does not imply hook installation. Starting the network is a separate, explicitly authorized action. -For source `c6846f6239329f0803142afc06c15a07b93937c1`, the accepted local-source Linux executable has -SHA256 `2bba8d58be24250bc262f75f835ce2d780369add275774f3f2135c623d23d29c`. -`bin/check-st2-pin-consistency.sh` keeps this active spec, the README, and executable preflight aligned. -`bin/check-corpus.sh` verifies the variable-age version contract, exact installed binary, strict semantic -validation, fixture resets, and the rest of the model-free corpus gate before an eval may run. +For source `c6846f6239329f0803142afc06c15a07b93937c1`, the accepted artifact is the immutable Nix package: + +- source `flake.lock` byte SHA256 + `aa547f85b21a8a8787adaa9f2a3ad37d55246d355248388e2ec38bd85a830141`; +- derivation `/nix/store/nrhfzsarya6ny7wdmy64i9vwgax965a8-st2-0.1.0.drv`; +- output `/nix/store/z4wj1y20wq00n02gpknnz45fdi2kyc7h-st2-0.1.0`, NAR + `sha256-w3OSHKt96U0aWMzW2FHom6Ii1NvikdXqmJ6wJ1xGJNA=` (5,147,208 bytes); +- version prefix `st2 0.1.0+c6846f6` and packaged executable SHA256 + `25276f30a9cfb287e1a9f72318fada2936402343e958160d46a0a7270ab2db2b`. + +`bin/check-st2-package-provenance.sh` verifies the whole chain and selects that exact binary rather than an +ambient `st2`. Its mutation gate rejects a same-version substitute. `bin/check-st2-pin-consistency.sh` keeps +this active spec, the README, and executable preflight aligned. `bin/check-corpus.sh` runs semantic validation +and every model-free gate with the pinned package before an eval may run. diff --git a/README.md b/README.md index f8512ebf..2d6fe53b 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,16 @@ SCORE: N PASS / 0 FAIL / K gating judges VERDICT: PASS ``` -Requirements are `st2 0.1.0` from source -[`c6846f6`](https://github.com/compoundingtech/st2/commit/c6846f6239329f0803142afc06c15a07b93937c1), -using the accepted local-source Linux binary SHA256 -`2bba8d58be24250bc262f75f835ce2d780369add275774f3f2135c623d23d29c`; -`pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and Node for JavaScript fixtures. A paid cell -also needs every harness named by its dry-run row. +Requirements are the immutable `st2 0.1.0+c6846f6` Nix package for source +[`c6846f6239329f0803142afc06c15a07b93937c1`](https://github.com/compoundingtech/st2/commit/c6846f6239329f0803142afc06c15a07b93937c1): +source `flake.lock` byte SHA256 +`aa547f85b21a8a8787adaa9f2a3ad37d55246d355248388e2ec38bd85a830141`, output +`/nix/store/z4wj1y20wq00n02gpknnz45fdi2kyc7h-st2-0.1.0`, NAR +`sha256-w3OSHKt96U0aWMzW2FHom6Ii1NvikdXqmJ6wJ1xGJNA=`, and executable SHA256 +`25276f30a9cfb287e1a9f72318fada2936402343e958160d46a0a7270ab2db2b`. +The preflight ignores ambient `st2` selection and verifies this full provenance chain before using the +packaged binary. It also requires Nix, `pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and +Node for JavaScript fixtures. A paid cell also needs every harness named by its dry-run row. ## Cell layout diff --git a/bin/check-canonical-agent-runtime-smoke.sh b/bin/check-canonical-agent-runtime-smoke.sh index 74725011..2807fbc9 100755 --- a/bin/check-canonical-agent-runtime-smoke.sh +++ b/bin/check-canonical-agent-runtime-smoke.sh @@ -5,17 +5,8 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" source bin/st2-pin.sh -st2_bin="$(command -v st2)" -version="$(st2 --version)" -sha256="$(sha256sum "$st2_bin" | awk '{ print $1 }')" -[[ "$version" =~ $ST2_VERSION_REGEX ]] || { - echo "FAIL: expected st2 source $ST2_SOURCE_SHORT, found $version" >&2 - exit 1 -} -[ "$sha256" = "$ST2_BINARY_SHA256" ] || { - echo "FAIL: expected st2 binary $ST2_BINARY_SHA256, found $sha256 at $st2_bin" >&2 - exit 1 -} +bin/check-st2-package-provenance.sh +export PATH="$ST2_OUTPUT_PATH/bin:$PATH" bin/corpus-inventory.sh --no-header | awk -F '\t' '$1 == "canonical-agent-runtime-smoke" && $2 == "model-free" && $5 == 0 { found = 1 } END { exit !found }' || { diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index 7e8063be..fdb15f64 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -7,18 +7,9 @@ cd "$repo_root" source bin/st2-pin.sh bin/check-st2-pin-consistency.sh -st2_path="$(command -v st2)" -actual_version="$(st2 --version)" -[[ "$actual_version" =~ $ST2_VERSION_REGEX ]] || { - echo "FAIL: expected st2 0.1.0 from pinned source $ST2_SOURCE_SHORT, found $actual_version" >&2 - exit 1 -} -actual_binary_sha256="$(sha256sum "$st2_path" | awk '{ print $1 }')" -[ "$actual_binary_sha256" = "$ST2_BINARY_SHA256" ] || { - echo "FAIL: expected st2 binary sha256 $ST2_BINARY_SHA256, found $actual_binary_sha256 at $st2_path" >&2 - exit 1 -} -echo "PASS: pinned runner source $ST2_SOURCE_FULL ($actual_version; sha256 $actual_binary_sha256)" +bin/check-st2-package-provenance.sh +bin/check-st2-package-provenance-mutations.sh +export PATH="$ST2_OUTPUT_PATH/bin:$PATH" mapfile -d '' shell_files < <( find bin cells -type f -name '*.sh' -not -path '*/_git/*' -print0 | sort -z diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index bd8e21ce..fe90cced 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -26,6 +26,8 @@ reachable=( bin/check-preflight-safety.sh bin/check-run-history.sh bin/check-retired-surfaces.sh + bin/check-st2-package-provenance-mutations.sh + bin/check-st2-package-provenance.sh bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh @@ -84,6 +86,8 @@ expected_direct=( bin/check-preflight-safety.sh bin/check-retired-surfaces.sh bin/check-run-history.sh + bin/check-st2-package-provenance-mutations.sh + bin/check-st2-package-provenance.sh bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh @@ -102,7 +106,6 @@ for file in "${reachable[@]}"; do while IFS= read -r hit; do line="${hit#*:}" case "$line" in - *'st2 --version'*) ;; *'st2 ls '*) ;; *'st2 validate '*) ;; *'st2 up '*'--materialize-only'*) ;; diff --git a/bin/check-st2-package-provenance-mutations.sh b/bin/check-st2-package-provenance-mutations.sh new file mode 100755 index 00000000..0c53549f --- /dev/null +++ b/bin/check-st2-package-provenance-mutations.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +source bin/st2-pin.sh + +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +bin/check-st2-package-provenance.sh >/dev/null + +mkdir -p "$scratch/other-st2-checkout/bin" +substitute="$scratch/other-st2-checkout/bin/st2" +cat >"$substitute" <