diff --git a/AGENT-SPEC.md b/AGENT-SPEC.md index f1e46481..de1f966e 100644 --- a/AGENT-SPEC.md +++ b/AGENT-SPEC.md @@ -1,8 +1,8 @@ # Canonical st2 agent specification This is the sole agent-authoring specification for this repository. It is pinned to st2 -[`9887b2842222def0838c2cd82e6c24c218f7efa6`](https://github.com/compoundingtech/st2/commit/9887b2842222def0838c2cd82e6c24c218f7efa6) -(`0.1.0`, source `9887b28`). It documents the hand-authored KDL accepted at that commit. Do not infer +[`c6846f6239329f0803142afc06c15a07b93937c1`](https://github.com/compoundingtech/st2/commit/c6846f6239329f0803142afc06c15a07b93937c1) +(`0.1.0`, source `c6846f6`). It documents the hand-authored KDL accepted at that commit. Do not infer additional fields or commands from older corpus fixtures. st2 runs long-lived `service` agents made of interactive `pty` tasks and terminal-free `exec` tasks. @@ -45,6 +45,7 @@ agent "" { supervisor "." retired #false keep #false + lifecycle "service" restart { attempts 3 @@ -57,7 +58,7 @@ agent "" { ST_AGENT "." } - command #""# + argv "" "" ding render { @@ -82,9 +83,11 @@ Supported agent children are: | `supervisor "…"` | Optional bare identity or full bus id for crash-loop routing. | | `retired #true` | Decommission the declaration on the next reconciliation. Edit this flag; do not delete a live declaration to retire it. | | `keep #true` | Freeze dead evidence and suppress collection/restart for every task; retirement still stops live tasks. | +| `lifecycle "service"|"adopt-only"` | Reconciliation policy for the compact task. Explicit tasks declare their own policy. | | `restart { … }` | Optional service restart policy. | | `env { KEY "value" }` | Environment inherited by the compact agent task and sidecars. | -| `command "…"` | Compact interactive task named `agent`. | +| `argv "program" "arg"…` | Preferred compact interactive task named `agent`; direct program invocation with preserved argument boundaries. | +| `command "…"` | Compact interactive shell task named `agent`; use only when shell syntax is intentional. | | `ding` | Compact native DING sidecar named `ding`. | | `pty "name" { … }` | Explicit interactive task. | | `exec "name" { … }` | Explicit non-interactive task. | @@ -102,20 +105,22 @@ rely on that permissiveness. ## Compact and explicit tasks -The canonical compact pair: +The canonical compact pair uses direct argv: ```kdl -command #""# +argv "" "" ding ``` lowers to an interactive `pty "agent"` and a non-interactive `exec "ding"` sidecar. Do not declare -both `command` and `pty "agent"`, or both `ding` and `exec "ding"`. +both compact `argv` and compact `command`; do not combine either compact launch with `pty "agent"`; +and do not declare both `ding` and `exec "ding"`. The compact task has id equal to the agent bus id, tag `role=agent`, and inherited agent environment. The derived DING task has id `.ding` and inherited environment. A service is runnable only when at least -one authored task has a command; the derived sidecar alone is insufficient, including on a retired -declaration. Task names are sorted lexically after lowering. +one authored task has exactly one `argv` or `command`; `argv` must have a non-empty program element. The +derived sidecar alone is insufficient, including on a retired declaration. Task names are sorted lexically +after lowering. Use explicit tasks only when the agent needs an additional managed process or task-specific configuration: @@ -123,9 +128,10 @@ configuration: ```kdl pty "agent" { id "." - command #""# + argv "" "" cwd "/absolute/workspace/or/$CATALOG/path" keep #false + lifecycle "service" tags role="agent" purpose="subject" env { ST_AGENT "." @@ -137,6 +143,7 @@ exec "helper" { command #"exec ./long-running-helper"# cwd "/absolute/workspace/or/$CATALOG/path" keep #false + lifecycle "adopt-only" tags purpose="fixture" env { EXAMPLE "value" @@ -144,11 +151,17 @@ exec "helper" { } ``` -Each explicit task supports only `id`, `command`, `cwd`, `keep`, `tags`, and `env`. A nameless task is a -validation error. Agent-level environment is parsed before tasks and inherited independent of declaration -order; task-level values override it. A missing `cwd` falls back to agent workspace, then the spec directory. -Commands run through `sh -c`. Interactive harnesses belong in `pty`; terminal-free daemons and helpers belong -in `exec`. +Each explicit task supports only `id`, `argv`, `command`, `cwd`, `keep`, `lifecycle`, `tags`, and `env`. +A nameless task, empty `argv`, or task carrying both `argv` and `command` is a validation error. Agent-level +environment is parsed before tasks and inherited independent of declaration order; task-level values override +it. A missing `cwd` falls back to agent workspace, then the spec directory. + +`argv` resolves a bare element zero through the task environment's `PATH`, expands st2 environment references +in each element, preserves argument boundaries, and launches the program directly without a shell. `command` +is opaque shell source run verbatim under `sh -c`; use it only for intentional shell behavior such as pipes, +redirections, compound commands, or shell expansion. Writing `exec` inside `command` can remove the inner +shell after it interprets that source, but it does not turn the declaration into direct argv. Interactive +harnesses belong in `pty`; terminal-free daemons and helpers belong in `exec`. Interactive tasks run detached through the PTY runtime with stable lifecycle ids. Exec tasks allocate no terminal, run in their own detached process group, append diagnostics below `/logs/`, and retain one @@ -159,7 +172,8 @@ what stops them. ## Environment and expansion Expansion recognizes `$VAR`, `${VAR}`, and `$$` for a literal dollar. Unset variables remain literal. Task -environment values, tags, and cwd expand before spawn; the command remains opaque for its `sh -c`. +environment values, tags, cwd, and each direct argv element expand before spawn; command remains opaque for +its `sh -c`. st2 defaults `CATALOG` and `ST_ROOT` to the catalog. Effective `PTY_ROOT` is a non-empty ambient value or `/pty`; authored task environment cannot override it. st2 supplies `ST_HOOKS` only when an installed @@ -225,13 +239,21 @@ misclassified as an empty registry. Reconciliation is per task: - active and alive: adopt; -- missing: launch only that task; -- dead non-keep: preserve bounded diagnostics, collect, and restart according to policy; +- missing service task: launch only that task; +- dead non-keep service task: preserve bounded diagnostics, collect, and restart according to policy; - dead keep: freeze evidence without collection or restart; +- missing or dead `adopt-only` task: report `held` without collection, replacement, or launch; - retired and live: stop even when keep is set; - retired and dead non-keep: final collection; - unrendered or unrunnable: do not launch. +`lifecycle "adopt-only"` is a migration fence, not a restart policy. st2 may adopt an already-live generation +but may not create a missing generation or reap and replace a dead one. Declare it at agent level for the +compact task or inside each applicable explicit task. Explicit tasks that omit it independently default to +`service`; they do not inherit the compact-task policy. Deliberately restore `service` (or remove the field) +to authorize normal launch/replacement. `retired #true` is the separate teardown authority and takes +precedence. + `role` has no branch in lifecycle behavior. `supervisor` is used for `ST_SUPERVISOR` and best-effort crash-loop routing. New exact Codex launches require verified installed hooks and workspace pretrust; already live/adopted Codex tasks are not stopped by that launch gate. @@ -267,7 +289,7 @@ agent "" { ST_AGENT "." } - command #"exec claude --model claude-sonnet-5 --effort medium --permission-mode bypassPermissions ''"# + argv "claude" "--model" "claude-sonnet-5" "--effort" "medium" "--permission-mode" "bypassPermissions" "" ding render { @@ -334,7 +356,7 @@ agent "" { ST_AGENT "." } - command #"exec codex --model gpt-5.6-sol -c 'model_reasoning_effort="medium"' --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust ''"# + argv "codex" "--model" "gpt-5.6-sol" "-c" "model_reasoning_effort=\"medium\"" "--dangerously-bypass-approvals-and-sandbox" "--dangerously-bypass-hook-trust" "" ding render { @@ -409,7 +431,7 @@ native DING wakeups. Structured exceptions are in `evidence/harness-exclusions.t At `agents/example/worker/agent.kdl`, path-derived identity and host make this the smallest valid service: ```kdl -agent { command "true" } +agent { argv "true" } ``` At the pinned source it validates as one agent with zero errors and warnings. Production declarations should @@ -421,7 +443,7 @@ example, create every `$CATALOG`-rooted workspace it names. ## Free authoring gate -These commands parse and materialize without starting a model seat: +These commands parse and materialize without starting a model agent: ```sh st2 validate --catalog "$CATALOG" --host --strict @@ -432,8 +454,17 @@ Inspect the declaration, every referenced template, and every workspace destinat materialization command. Materialization is byte-idempotent and does not imply hook installation. Starting the network is a separate, explicitly authorized action. -For source `9887b28`, the accepted Linux executable has SHA256 -`d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d`; its published archive has SHA256 -`32ee103bd17ccb3e155ac63d816a3906c2470a3c98e3cc04b56e5a67138b9927`. `bin/check-corpus.sh` verifies -the variable-age version contract, exact installed binary, embedded full source commit, strict semantic -validation, fixture resets, and the rest of the model-free corpus gate before an eval may run. +For source `c6846f6239329f0803142afc06c15a07b93937c1`, the accepted artifact is the immutable Nix package: + +- source `flake.lock` byte SHA256 + `aa547f85b21a8a8787adaa9f2a3ad37d55246d355248388e2ec38bd85a830141`; +- derivation `/nix/store/nrhfzsarya6ny7wdmy64i9vwgax965a8-st2-0.1.0.drv`; +- output `/nix/store/z4wj1y20wq00n02gpknnz45fdi2kyc7h-st2-0.1.0`, NAR + `sha256-w3OSHKt96U0aWMzW2FHom6Ii1NvikdXqmJ6wJ1xGJNA=` (5,147,208 bytes); +- version prefix `st2 0.1.0+c6846f6` and packaged executable SHA256 + `25276f30a9cfb287e1a9f72318fada2936402343e958160d46a0a7270ab2db2b`. + +`bin/check-st2-package-provenance.sh` verifies the whole chain and selects that exact binary rather than an +ambient `st2`. Its mutation gate rejects a same-version substitute. `bin/check-st2-pin-consistency.sh` keeps +this active spec, the README, and executable preflight aligned. `bin/check-corpus.sh` runs semantic validation +and every model-free gate with the pinned package before an eval may run. diff --git a/CATALOG.md b/CATALOG.md index 58cd2eb1..916c74c3 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,26 +1,34 @@ # Current eval catalog and evidence -This is the sole current corpus surface: **45 included cells** (14 model-free, -31 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active +This is the sole current corpus surface: **53 included cells** (18 model-free, +35 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active KDL and executable judge scripts. Latest accepted PASS evidence remains distinct from the append-only run history: a failed last run is visible without being advertised as accepted evidence, and cells with no structured run row say so explicitly. Every model launch is explicitly pinned by `bin/check-model-policy.sh`: Claude uses `claude-sonnet-5` at medium effort and Codex uses `gpt-5.6-sol` at medium reasoning effort. -Every bus-connected model seat is mechanically checked for the event-first DING lifecycle and its +Every bus-connected model agent is mechanically checked for the event-first DING lifecycle and its harness-native loader plus canonical hook file. ## Included overnight inventory -| Cell | Harness | Model(s) / effort | Model seats | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | +| Cell | Harness | Model(s) / effort | Model agents | Cost | Timeout | Held-out judges | Latest accepted PASS | Last recorded run | |---|---|---|---:|---|---|---:|---|---| +| `agent-new-bundle-contract` | model-free | — | 0 | none | `60s` | 6 | — | **NO STRUCTURED RUN** | +| `agent-new-confirmation-contract` | model-free | — | 0 | none | `30s` | 4 | — | **NO STRUCTURED RUN** | +| `agent-new-creation-lifecycle` | model-free | — | 0 | none | `180s` | 5 | — | **NO STRUCTURED RUN** | +| `agent-new-interview` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 5 | — | **NO STRUCTURED RUN** | +| `agent-new-interview-hard-constraint` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | +| `agent-new-interview-investigation` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | +| `agent-new-interview-small-fix` | Claude | `claude-sonnet-5` / medium | 1 | low | `300s` | 4 | — | **NO STRUCTURED RUN** | +| `canonical-agent-runtime-smoke` | model-free | — | 0 | none | `30s` | 7 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `ding-reply` | Claude | `claude-sonnet-5` / medium | 1 | low | `420s` | 2 | — | **NO STRUCTURED RUN** | -| `docs` | Claude | `claude-sonnet-5` / medium | 3 | high | `1200s` | 5 | — | **NO STRUCTURED RUN** | +| `docs` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 5 | — | **NO STRUCTURED RUN** | | `exec-lifecycle-logging` | model-free | — | 0 | none | `90s` | 5 | — | **NO STRUCTURED RUN** | | `feature-fit` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 5 | — | **NO STRUCTURED RUN** | | `fork-in-the-road` | Claude | `claude-sonnet-5` / medium | 4 | high | `1200s` | 5 | — | **NO STRUCTURED RUN** | @@ -74,30 +82,34 @@ while the last-run column makes a recorded failure distinct from a cell with no - `clean-compose` — retired composition-overlay discriminator - `compose-config-load` — retired composition/config-loader discriminator - `compose-global-skill` — retired composition/global-skill discriminator -- `team-standup` — retired runtime seat-generation discriminator +- `team-standup` — retired runtime agent-generation discriminator `hook-integrity` is not excluded: it is now a model-free, current-native `st2 up --materialize-only` hook installation and idempotency test. ## Harness hook exclusions -- `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness seat +- `agent-new-bundle-contract` / `cell` — deterministic semantic-intent lowering and mutation probe; no harness agent +- `agent-new-confirmation-contract` / `cell` — deterministic pre-threshold human-confirmation boundary; no harness agent +- `agent-new-creation-lifecycle` / `cell` — successor-gated real Axe lifecycle matrix with no provider model +- `canonical-agent-runtime-smoke` / `cell` — deterministic canonical Agent Spec boot, kickoff, reply, and teardown probe; no harness agent +- `context-resource-continuity` / `cell` — deterministic native context/resource restart probe; no harness agent - `docs` / `judge:cold-reader` — one-shot offline Claude print grader; no bus identity, DING, or hook surface -- `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness seat -- `hook-integrity` / `cell` — deterministic native hook-installation probe; no live harness seat -- `host-lock-health-negatives` / `cell` — deterministic native doctor negative probe; no harness seat -- `pty-attach-only` / `cell` — deterministic PTY dead-attach policy probe; no harness seat -- `pty-send-peek` / `cell` — deterministic PTY transport probe; no harness seat -- `presence-ding-matrix` / `cell` — deterministic native presence and DING probe; no harness seat -- `reconcile-retire-keep` / `cell` — deterministic native reconcile lifecycle probe; no harness seat -- `render-target-safety` / `cell` — deterministic native render-policy probe; no harness seat -- `strict-validation-json` / `cell` — deterministic native validation-diagnostics probe; no harness seat -- `st2-doctor-structure` / `cell` — deterministic catalog-health probe; no harness seat -- `st2-network` / `cell` — deterministic catalog/message round-trip probe; no harness seat +- `exec-lifecycle-logging` / `cell` — deterministic native exec lifecycle probe; no harness agent +- `hook-integrity` / `cell` — deterministic native hook-installation probe; no live harness agent +- `host-lock-health-negatives` / `cell` — deterministic native doctor negative probe; no harness agent +- `pty-attach-only` / `cell` — deterministic PTY dead-attach policy probe; no harness agent +- `pty-send-peek` / `cell` — deterministic PTY transport probe; no harness agent +- `presence-ding-matrix` / `cell` — deterministic native presence and DING probe; no harness agent +- `reconcile-retire-keep` / `cell` — deterministic native reconcile lifecycle probe; no harness agent +- `render-target-safety` / `cell` — deterministic native render-policy probe; no harness agent +- `strict-validation-json` / `cell` — deterministic native validation-diagnostics probe; no harness agent +- `st2-doctor-structure` / `cell` — deterministic catalog-health probe; no harness agent +- `st2-network` / `cell` — deterministic catalog/message round-trip probe; no harness agent - `targeted-reconcile-isolation` / `cell` — deterministic selected reconcile isolation probe; no bus-connected agent -- `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness seat +- `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness agent -These **15** rows are generated from structured exclusions. A cell-level row is required for +These **19** rows are generated from structured exclusions. A cell-level row is required for every derived model-free cell. The one-shot docs cold reader is a model-backed grader, but not a bus-connected agent: it has no st2 identity, DING sidecar, or lifecycle hooks. @@ -109,7 +121,7 @@ Run the complete model-free preflight: bin/check-corpus.sh ``` -Preview the exact lexical run order, harness, model, effort, seat count, cost band, and timeout: +Preview the exact lexical run order, harness, model, effort, agent count, cost band, and timeout: ```sh bin/overnight.sh --dry-run diff --git a/README.md b/README.md index 37808f25..2d6fe53b 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ next paid cell to start. ## Run one cell `st2 eval` creates a hermetic temporary catalog, copies the fixture, boots declared agents and model judges, -delivers the kickoff, waits for completion or the cell timeout, tears seats down, and runs the held-out judges: +delivers the kickoff, waits for completion or the cell timeout, tears agents down, and runs the held-out judges: ```sh st2 eval ./cells/ghost-bug/ @@ -95,10 +95,16 @@ SCORE: N PASS / 0 FAIL / K gating judges VERDICT: PASS ``` -Requirements are `st2 0.1.0` from source -[`9887b28`](https://github.com/compoundingtech/st2/commit/9887b2842222def0838c2cd82e6c24c218f7efa6), -`pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and Node for JavaScript fixtures. A paid cell -also needs every harness named by its dry-run row. +Requirements are the immutable `st2 0.1.0+c6846f6` Nix package for source +[`c6846f6239329f0803142afc06c15a07b93937c1`](https://github.com/compoundingtech/st2/commit/c6846f6239329f0803142afc06c15a07b93937c1): +source `flake.lock` byte SHA256 +`aa547f85b21a8a8787adaa9f2a3ad37d55246d355248388e2ec38bd85a830141`, output +`/nix/store/z4wj1y20wq00n02gpknnz45fdi2kyc7h-st2-0.1.0`, NAR +`sha256-w3OSHKt96U0aWMzW2FHom6Ii1NvikdXqmJ6wJ1xGJNA=`, and executable SHA256 +`25276f30a9cfb287e1a9f72318fada2936402343e958160d46a0a7270ab2db2b`. +The preflight ignores ambient `st2` selection and verifies this full provenance chain before using the +packaged binary. It also requires Nix, `pty`, Bash, Git, `jq`, Rust/Cargo for the pinned KDL parser gate, and +Node for JavaScript fixtures. A paid cell also needs every harness named by its dry-run row. ## Cell layout diff --git a/bin/check-agent-new-behavior-cases.sh b/bin/check-agent-new-behavior-cases.sh new file mode 100755 index 00000000..54a7f356 --- /dev/null +++ b/bin/check-agent-new-behavior-cases.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +cases_root="cells/agent-new-interview/fixture/cases" +cases=(hard-constraint implementation investigation small-fix) +model_context=( + cells/agent-new-interview/fixture/interviewer/CLAUDE.md + cells/agent-new-interview/fixture/interviewer/PERSONA.md + cells/agent-new-interview/fixture/interviewer/PROTOCOL.md +) + +for case_name in "${cases[@]}"; do + case_root="$cases_root/$case_name" + task="$case_root/task.md" + constraints="$case_root/hard-constraints.json" + expected="$case_root/expected.json" + test -f "$task" && test -f "$constraints" && test -f "$expected" + test "$(awk 'NF { rows += 1 } END { print rows + 0 }' "$task")" -eq 1 + jq -e ' + type == "object" and + (.identity | test("^[a-z0-9-]+\\.[a-z0-9-]+\\.[a-z0-9-]+\\.[a-z0-9-]+$")) and + (.goalTerms | type == "array" and length > 0) and + (.references | type == "array") and + (.trajectory | keys | sort) == ([ + "boot", "effort", "harness", "mode", "model", "persona" + ] | sort) + ' "$expected" >/dev/null + jq -e ' + type == "object" and + (keys | all(. == "harness" or . == "model" or . == "effort" or + . == "persona" or . == "supervisor")) + ' "$constraints" >/dev/null + identity="$(jq -r .identity "$expected")" + if rg -Fq "$identity" "${model_context[@]}"; then + echo "FAIL: held-out identity for $case_name leaked into model-facing context" >&2 + exit 1 + fi +done + +test "$(jq '.references | length' "$cases_root/small-fix/expected.json")" -eq 0 +test "$(jq 'length' "$cases_root/hard-constraint/hard-constraints.json")" -ge 5 + +for cell in \ + agent-new-interview \ + agent-new-interview-hard-constraint \ + agent-new-interview-investigation \ + agent-new-interview-small-fix; do + test -f "cells/$cell/$cell.kdl" +done + +echo "PASS: four one-sentence paid cases keep expected decisions held out and hard constraints explicit" diff --git a/bin/check-agent-new-interview-attempts-mutations.sh b/bin/check-agent-new-interview-attempts-mutations.sh new file mode 100755 index 00000000..b6b577e6 --- /dev/null +++ b/bin/check-agent-new-interview-attempts-mutations.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Prove the append-only helper accepts an append and rejects a baseline-row rewrite. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +scratch="$(mktemp -d)" +trap 'rm -rf -- "$scratch"' EXIT + +git -C "$scratch" init -q --initial-branch=main +mkdir -p "$scratch/evidence" +printf 'header\nbaseline-row\n' >"$scratch/evidence/attempts.tsv" +git -C "$scratch" add evidence/attempts.tsv +GIT_AUTHOR_DATE="2000-01-01T00:00:00Z" \ +GIT_COMMITTER_DATE="2000-01-01T00:00:00Z" \ + git -C "$scratch" \ + -c core.hooksPath=/dev/null \ + -c user.name=eval-fixture \ + -c user.email=eval-fixture@local \ + commit -q -m "fixture: append-only baseline" +baseline="$(git -C "$scratch" rev-parse HEAD)" + +printf 'appended-row\n' >>"$scratch/evidence/attempts.tsv" +"$repo_root/bin/check-append-only-file.sh" \ + --repo-root "$scratch" --baseline "$baseline" --path evidence/attempts.tsv >/dev/null + +printf 'header\nrewritten-row\nappended-row\n' >"$scratch/evidence/attempts.tsv" +if "$repo_root/bin/check-append-only-file.sh" \ + --repo-root "$scratch" --baseline "$baseline" --path evidence/attempts.tsv >/dev/null 2>&1; then + echo "FAIL: append-only helper accepted a rewritten baseline row" >&2 + exit 1 +fi + +echo "PASS: append-only mutation accepts appended bytes and rejects a baseline-row rewrite" diff --git a/bin/check-agent-new-interview-attempts.sh b/bin/check-agent-new-interview-attempts.sh new file mode 100755 index 00000000..04cce282 --- /dev/null +++ b/bin/check-agent-new-interview-attempts.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Validate pre-run evidence without representing it as a judged model run. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +ledger="evidence/agent-new-interview-attempts.tsv" +baseline="" +if [ "$#" -eq 2 ] && [ "$1" = "--baseline" ]; then + baseline="$2" +elif [ "$#" -ne 0 ]; then + echo "usage: $0 [--baseline COMMIT]" >&2 + exit 2 +fi +expected_header=$'recorded_at_utc\tsource_commit\tst2_commit\tst2_sha256\taxe_source_commit\tadapter\truntime_profile\tstage\tresult\tdiagnostic\tcleanup' + +[ -f "$ledger" ] && [ "$(head -n 1 "$ledger")" = "$expected_header" ] || { + echo "FAIL: $ledger is missing or has an unexpected header" >&2 + exit 1 +} + +if [ -n "$baseline" ]; then + bin/check-append-only-file.sh \ + --repo-root "$repo_root" --baseline "$baseline" --path "$ledger" +fi + +rows=0 +while IFS=$'\t' read -r recorded source st2_commit st2_sha axe_head adapter profile stage result diagnostic cleanup extra; do + [ "$recorded" != "recorded_at_utc" ] || continue + [ -n "$recorded" ] || continue + [ -z "${extra:-}" ] || { echo "FAIL: $ledger row has extra columns" >&2; exit 1; } + [[ "$recorded" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || + { echo "FAIL: invalid recorded_at_utc $recorded" >&2; exit 1; } + [[ "$source" =~ ^[0-9a-f]{40}$ ]] && git cat-file -e "$source^{commit}" 2>/dev/null || + { echo "FAIL: unavailable source commit $source" >&2; exit 1; } + [[ "$st2_commit" =~ ^[0-9a-f]{40}$ && "$st2_sha" =~ ^[0-9a-f]{64}$ ]] || + { echo "FAIL: st2 identity is not exact" >&2; exit 1; } + [[ "$axe_head" =~ ^[0-9a-f]{40}$ && "$adapter" == /nix/store/* && "$profile" == /nix/store/* ]] || + { echo "FAIL: Axe/runtime artifact identity is not exact" >&2; exit 1; } + [ "$stage" = "materialization" ] && [ "$result" = "PRE_AXE_FAIL" ] || + { echo "FAIL: pre-run evidence claims an unsupported stage or result" >&2; exit 1; } + [ -n "$diagnostic" ] && [ "$cleanup" = "catalog-absent;process-absent" ] || + { echo "FAIL: pre-run evidence lacks diagnostic or cleanup proof" >&2; exit 1; } + ((rows += 1)) +done <"$ledger" + +[ "$rows" -gt 0 ] || { echo "FAIL: $ledger has no evidence rows" >&2; exit 1; } +if [ -n "$baseline" ]; then + printf 'PASS: %d pre-Axe attempt row(s) validated with exact artifacts, cleanup, and an immutable baseline\n' "$rows" +else + printf 'PASS: %d pre-Axe attempt row(s) validated with exact artifacts and cleanup; no append-only baseline supplied\n' "$rows" +fi diff --git a/bin/check-agent-new-renderer-security.sh b/bin/check-agent-new-renderer-security.sh new file mode 100755 index 00000000..c689fa88 --- /dev/null +++ b/bin/check-agent-new-renderer-security.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +input="$repo_root/cells/agent-new-bundle-contract/fixture/inputs/workspace-injection.json" +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +renderers=( + "$repo_root/cells/agent-new-bundle-contract/fixture/render-intent.sh|$repo_root/cells/agent-new-bundle-contract/fixture/inputs/implementation-constraints.json" + "$repo_root/cells/agent-new-interview/fixture/interviewer/render-intent.sh|$repo_root/cells/agent-new-interview/fixture/cases/implementation/hard-constraints.json" +) + +for entry in "${renderers[@]}"; do + renderer="${entry%%|*}" + constraints="${entry#*|}" + name="$(basename "$(dirname "$renderer")")" + out="$scratch/$name" + if bash "$renderer" "$input" "$out" /workspace/dotfiles "$constraints" >/dev/null 2>&1; then + echo "FAIL: $renderer accepted a workspace KDL injection" >&2 + exit 1 + fi + [ ! -e "$out/agents" ] || { + echo "FAIL: $renderer emitted an Agent Spec after rejecting workspace KDL injection" >&2 + exit 1 + } +done + +echo "PASS: both Agent Spec renderers reject workspace KDL injection before output" diff --git a/bin/check-append-only-file.sh b/bin/check-append-only-file.sh new file mode 100755 index 00000000..a2737391 --- /dev/null +++ b/bin/check-append-only-file.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Prove that a working-tree file preserves an immutable baseline blob as an exact prefix. +set -euo pipefail + +repo_root="" +baseline="" +path="" +while [ "$#" -gt 0 ]; do + case "$1" in + --repo-root) repo_root="${2:-}"; shift 2 ;; + --baseline) baseline="${2:-}"; shift 2 ;; + --path) path="${2:-}"; shift 2 ;; + *) echo "usage: $0 --repo-root DIR --baseline REF --path RELATIVE_PATH" >&2; exit 2 ;; + esac +done +[ -n "$repo_root" ] && [ -n "$baseline" ] && [ -n "$path" ] || + { echo "usage: $0 --repo-root DIR --baseline REF --path RELATIVE_PATH" >&2; exit 2; } +[[ "$path" != /* && "$path" != *..* ]] || + { echo "FAIL: append-only path must be repository-relative" >&2; exit 1; } + +baseline_commit="$(git -C "$repo_root" rev-parse --verify "$baseline^{commit}")" || + { echo "FAIL: append-only baseline is not an immutable commit: $baseline" >&2; exit 1; } +git -C "$repo_root" cat-file -e "$baseline_commit:$path" 2>/dev/null || + { echo "FAIL: baseline $baseline_commit has no $path to compare" >&2; exit 1; } +[ -f "$repo_root/$path" ] || { echo "FAIL: working file is missing: $path" >&2; exit 1; } + +prior="$(mktemp)" +trap 'rm -f -- "$prior"' EXIT +git -C "$repo_root" show "$baseline_commit:$path" >"$prior" +prior_size="$(wc -c <"$prior")" +current_size="$(wc -c <"$repo_root/$path")" +[ "$current_size" -ge "$prior_size" ] && + cmp -n "$prior_size" "$prior" "$repo_root/$path" || { + echo "FAIL: $path rewrites or removes baseline $baseline_commit; only append bytes" >&2 + exit 1 + } +printf 'PASS: %s preserves baseline %s as an exact prefix\n' "$path" "$baseline_commit" diff --git a/bin/check-canonical-agent-runtime-smoke.sh b/bin/check-canonical-agent-runtime-smoke.sh new file mode 100755 index 00000000..2807fbc9 --- /dev/null +++ b/bin/check-canonical-agent-runtime-smoke.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +source bin/st2-pin.sh + +bin/check-st2-package-provenance.sh +export PATH="$ST2_OUTPUT_PATH/bin:$PATH" + +bin/corpus-inventory.sh --no-header | + awk -F '\t' '$1 == "canonical-agent-runtime-smoke" && $2 == "model-free" && $5 == 0 { found = 1 } END { exit !found }' || { + echo "FAIL: canonical-agent-runtime-smoke is not classified as zero-model-agent model-free" >&2 + exit 1 + } + +log="$(mktemp)" +catalog="" +cleanup() { + rm -f -- "$log" + if [ -n "$catalog" ]; then + case "$catalog" in + /tmp/st2e-[0-9]*) rm -rf -- "$catalog" ;; + *) echo "REFUSE: unexpected scratch catalog path $catalog" >&2 ;; + esac + fi +} +trap cleanup EXIT + +set +e +st2 eval ./cells/canonical-agent-runtime-smoke/ --keep >"$log" 2>&1 & +eval_pid=$! +catalog="/tmp/st2e-$eval_pid" +wait "$eval_pid" +status=$? +set -e +cat "$log" + +[ "$status" -eq 0 ] || { + echo "FAIL: canonical Agent Spec smoke exited $status" >&2 + exit "$status" +} +grep -Fq "SCORE: 9 PASS / 0 FAIL / 9 gating judges" "$log" +grep -Fq "VERDICT: PASS" "$log" + +for pid_file in "$catalog/probe/work.pid" "$catalog/probe/guard.pid"; do + pid="$(cat "$pid_file")" + [[ "$pid" =~ ^[0-9]+$ ]] || { + echo "FAIL: canonical runtime task receipt is not a PID: $pid_file=$pid" >&2 + exit 1 + } + if kill -0 "$pid" 2>/dev/null; then + echo "FAIL: canonical runtime task $pid from $pid_file survived eval teardown" >&2 + exit 1 + fi +done + +sessions="$(st2 pty --catalog "$catalog" ls --json)" +jq -e 'length == 0' <<<"$sessions" >/dev/null || { + echo "FAIL: canonical eval left PTY registry residue: $sessions" >&2 + exit 1 +} +test ! -e "$catalog/REMOTE-SPAWNED" + +echo "PASS: path-independent local PTY+exec launched, native routing completed, remote sentinel stayed inert, and teardown left zero tasks" diff --git a/bin/check-canonical-agent-template-mutations.sh b/bin/check-canonical-agent-template-mutations.sh new file mode 100755 index 00000000..438755e2 --- /dev/null +++ b/bin/check-canonical-agent-template-mutations.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +checker="$repo_root/bin/check-canonical-agent-template.sh" +source_template="$repo_root/cells/agent-new-interview/fixture/agent.kdl.template" +source_publisher="$repo_root/cells/agent-new-interview/fixture/publish-interviewer.sh" +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +cp "$source_template" "$scratch/valid.template" +cp "$source_publisher" "$scratch/publisher.sh" +"$checker" "$scratch/valid.template" "$scratch/publisher.sh" >/dev/null + +cp "$source_template" "$scratch/missing-boot.template" +sed -i 's/"--boot" "managed-v1"//' "$scratch/missing-boot.template" +if "$checker" "$scratch/missing-boot.template" "$scratch/publisher.sh" >/dev/null 2>&1; then + echo "FAIL: canonical template without managed-v1 boot passed" >&2 + exit 1 +fi + +cp "$source_template" "$scratch/missing-overlay.template" +sed -i '/copy "_templates\/bus.st2.md" ".st2\/bus.md"/d' "$scratch/missing-overlay.template" +if "$checker" "$scratch/missing-overlay.template" "$scratch/publisher.sh" >/dev/null 2>&1; then + echo "FAIL: canonical template without bus overlay passed" >&2 + exit 1 +fi + +echo "PASS: removing a required launch axis or canonical overlay fails the template gate" diff --git a/bin/check-canonical-agent-template.sh b/bin/check-canonical-agent-template.sh new file mode 100755 index 00000000..53a14f2a --- /dev/null +++ b/bin/check-canonical-agent-template.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Validate the canonical paid model-agent declaration template and publisher. +set -euo pipefail + +template="${1:?canonical Agent Spec template required}" +publisher="${2:?canonical Agent Spec publisher required}" + +require_once() { + local pattern="$1" description="$2" + if [ "$(grep -Fxc "$pattern" "$template" || true)" -ne 1 ]; then + echo "FAIL: $template must contain exactly one $description" >&2 + exit 1 + fi +} + +require_once ' AGENT_LAUNCH_HOSTED "1"' "hosted launch marker" +require_once ' AGENT_PERSONA "session-creator"' "canonical persona" +require_once ' AGENT_RUNTIME_PROFILE "@PROFILE@"' "runtime-profile projection" +require_once ' argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--model" "claude-sonnet-5" "--effort" "medium" "--persona" "session-creator" "--mode" "managed-unattended" "--boot" "managed-v1"' "typed managed-v1 launch" +require_once ' copy "_templates/session-creator.md" ".st2/PERSONA.md"' "canonical persona overlay" +require_once ' copy "_templates/bus.st2.md" ".st2/bus.md"' "canonical bus overlay" +require_once ' git-exclude ".st2/"' "overlay exclusion" +require_once ' ding' "native DING declaration" + +if grep -Fq -- '--account' "$template"; then + echo "FAIL: $template durably pins an account" >&2 + exit 1 +fi + +for projection in \ + 'profile="${AGENT_RUNTIME_PROFILE:-$HOME/.config/coding-agents/profile.json}"' \ + 'case "$profile" in' \ + '/*) ;;' \ + '.agentSpec.adapterBin' \ + '.personas.prompts["session-creator"]' \ + 's|@PROFILE@|$profile|' \ + 's|@ADAPTER@|$adapter|' \ + 'cp "$persona_source" ./_templates/session-creator.md'; do + grep -Fq "$projection" "$publisher" || { + echo "FAIL: $publisher omits canonical projection $projection" >&2 + exit 1 + } +done + +echo "PASS: $template is an account-neutral managed-v1 canonical template with explicit profile and overlay projection" diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index fc85e69d..24054b52 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -1,30 +1,15 @@ #!/usr/bin/env bash -# Complete free preflight for the current overnight corpus. Never starts a model seat. +# Complete free preflight for the current overnight corpus. Never starts a model agent. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -expected_source="9887b28" -expected_source_full="9887b2842222def0838c2cd82e6c24c218f7efa6" -expected_binary_sha256="d49d44fd4f3f6f655455c212353a469fefa956082bedf22163deb767d8a36a0d" -expected_version_regex='^st2 0\.1\.0 — running from local source \(9887b28, .+ ago\)$' -st2_path="$(command -v st2)" -actual_version="$(st2 --version)" -[[ "$actual_version" =~ $expected_version_regex ]] || { - echo "FAIL: expected st2 0.1.0 from pinned source $expected_source, found $actual_version" >&2 - exit 1 -} -actual_binary_sha256="$(sha256sum "$st2_path" | awk '{ print $1 }')" -[ "$actual_binary_sha256" = "$expected_binary_sha256" ] || { - echo "FAIL: expected st2 binary sha256 $expected_binary_sha256, found $actual_binary_sha256 at $st2_path" >&2 - exit 1 -} -LC_ALL=C grep -aFq "$expected_source_full" "$st2_path" || { - echo "FAIL: st2 binary at $st2_path does not embed full pinned source $expected_source_full" >&2 - exit 1 -} -echo "PASS: pinned runner source $expected_source ($actual_version; sha256 $actual_binary_sha256)" +source bin/st2-pin.sh +bin/check-st2-pin-consistency.sh +bin/check-st2-package-provenance.sh +bin/check-st2-package-provenance-mutations.sh +export PATH="$ST2_OUTPUT_PATH/bin:$PATH" mapfile -d '' shell_files < <( find bin cells -type f -name '*.sh' -not -path '*/_git/*' -print0 | sort -z @@ -33,11 +18,29 @@ bash -n "${shell_files[@]}" echo "PASS: ${#shell_files[@]} shell files parse" bin/check-preflight-safety.sh +bin/check-agent-new-behavior-cases.sh +bin/check-agent-new-renderer-security.sh bin/check-model-policy.sh bin/check-model-policy-mutations.sh +bin/check-canonical-agent-template-mutations.sh bin/check-run-history.sh +attempt_baseline="${EVALS_EVIDENCE_BASELINE_REF:-}" +if [ -z "$attempt_baseline" ] && git rev-parse --verify origin/main^{commit} >/dev/null 2>&1; then + candidate_baseline="$(git merge-base HEAD origin/main)" + current_commit="$(git rev-parse HEAD)" + if [ "$candidate_baseline" != "$current_commit" ] && + git cat-file -e "$candidate_baseline:evidence/agent-new-interview-attempts.tsv" 2>/dev/null; then + attempt_baseline="$candidate_baseline" + fi +fi +if [ -n "$attempt_baseline" ]; then + bin/check-agent-new-interview-attempts.sh --baseline "$attempt_baseline" +else + bin/check-agent-new-interview-attempts.sh +fi +bin/check-agent-new-interview-attempts-mutations.sh bin/check-retired-surfaces.sh -bin/model-seat-inventory.sh >/dev/null +bin/model-agent-inventory.sh >/dev/null bin/check-event-first.sh bin/check-kdl-parse.sh bin/check-st2-semantic.sh diff --git a/bin/check-event-first.sh b/bin/check-event-first.sh index f47d96d4..cd681b78 100755 --- a/bin/check-event-first.sh +++ b/bin/check-event-first.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Enforce one cold-start drain followed by native DING wakeups for every maintained model seat. +# Enforce one cold-start drain followed by native DING wakeups for every maintained model agent. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -12,7 +12,7 @@ cleanup() { } trap cleanup EXIT -bin/model-seat-inventory.sh --no-header > "$inventory" +bin/model-agent-inventory.sh --no-header > "$inventory" failed=0 fail() { @@ -29,12 +29,26 @@ required=( "report completion or blockers over the st2 bus" ) -while IFS=$'\t' read -r cell agent _harness _workspace _st_agent command_line; do - kdl="cells/$cell/$cell.kdl" - command_text="$(sed -n "${command_line}p" "$kdl")" +while IFS=$'\t' read -r cell agent _harness _workspace _st_agent source_kind source_path source_line; do + command_text="$(sed -n "${source_line}p" "$source_path")" + if [ "$source_kind" = "canonical-template" ]; then + grep -Fq '"--mode" "managed-unattended"' <<< "$command_text" || + fail "$source_path:$source_line agent $agent canonical launch omits managed-unattended mode" + grep -Fq '"--boot" "managed-v1"' <<< "$command_text" || + fail "$source_path:$source_line agent $agent canonical launch omits managed-v1 event-first boot contract" + continue + fi + kdl="$source_path" + if grep -Fq 'exec axe agent launch ' <<< "$command_text"; then + grep -Fq -- '--mode managed-unattended' <<< "$command_text" || + fail "$kdl:$source_line agent $agent Axe launch omits managed-unattended mode" + grep -Fq -- '--boot managed-v1' <<< "$command_text" || + fail "$kdl:$source_line agent $agent Axe launch omits managed-v1 event-first boot contract" + continue + fi for phrase in "${required[@]}"; do grep -Fiq "$phrase" <<< "$command_text" || - fail "$kdl:$command_line agent $agent does not teach '$phrase'" + fail "$kdl:$source_line agent $agent does not teach '$phrase'" done done < "$inventory" @@ -55,7 +69,7 @@ if rg -n --pcre2 \ fi if [ "$failed" -eq 0 ]; then - printf 'PASS: %s model seats use one cold-start drain and event-first native DING wakeups\n' \ + printf 'PASS: %s model agents use one cold-start drain and event-first native DING wakeups\n' \ "$(wc -l < "$inventory" | tr -d ' ')" fi exit "$failed" diff --git a/bin/check-fixture-reset-terminal.sh b/bin/check-fixture-reset-terminal.sh index 61bbba9e..272ff2e3 100755 --- a/bin/check-fixture-reset-terminal.sh +++ b/bin/check-fixture-reset-terminal.sh @@ -18,17 +18,35 @@ trap cleanup EXIT plain="$scratch/plain.out" terminal="$scratch/terminal.out" +hostile="$scratch/hostile.out" +hostile_hooks="$scratch/hostile-hooks" +hostile_config="$scratch/hostile.gitconfig" bin/check-fixture-reset.sh >"$plain" CHECK_FIXTURE_RESET="$repo_root/bin/check-fixture-reset.sh" \ script -qefc 'exec "$CHECK_FIXTURE_RESET"' /dev/null "$terminal" +mkdir -p "$hostile_hooks" +printf '%s\n' '#!/bin/sh' 'echo ambient-hook-ran >&2' 'exit 97' \ + >"$hostile_hooks/commit-msg" +chmod +x "$hostile_hooks/commit-msg" +git config -f "$hostile_config" core.hooksPath "$hostile_hooks" +git config -f "$hostile_config" user.name ambient-user +git config -f "$hostile_config" user.email ambient@example.invalid +GIT_CONFIG_GLOBAL="$hostile_config" GIT_CONFIG_NOSYSTEM=0 \ + bin/check-fixture-reset.sh >"$hostile" + if ! cmp -s "$plain" "$terminal"; then diff -u "$plain" "$terminal" >&2 || true echo "FAIL: fixture-reset gate differs with a terminal attached" >&2 exit 1 fi +if ! cmp -s "$plain" "$hostile"; then + diff -u "$plain" "$hostile" >&2 || true + echo "FAIL: fixture reset depends on ambient Git config or hooks" >&2 + exit 1 +fi sed -n '1,240p' "$plain" -echo "PASS: fixture-reset gate and owned-mktemp cleanup are terminal-independent" +echo "PASS: fixture-reset gate is terminal-independent and ignores hostile ambient Git config/hooks" diff --git a/bin/check-harness-contract.sh b/bin/check-harness-contract.sh index e03e1a68..284e2ad2 100755 --- a/bin/check-harness-contract.sh +++ b/bin/check-harness-contract.sh @@ -6,14 +6,19 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" selected="all" +selected_cell="" if [ "$#" -eq 2 ] && [ "$1" = "--harness" ]; then selected="$2" case "$selected" in Claude|Codex) ;; *) echo "usage: bin/check-harness-contract.sh [--harness Claude|Codex]" >&2; exit 2 ;; esac +elif [ "$#" -eq 2 ] && [ "$1" = "--cell" ]; then + selected_cell="$2" + [ -d "cells/$selected_cell" ] || + { echo "FAIL: unknown cell $selected_cell" >&2; exit 2; } elif [ "$#" -ne 0 ]; then - echo "usage: bin/check-harness-contract.sh [--harness Claude|Codex]" >&2 + echo "usage: bin/check-harness-contract.sh [--harness Claude|Codex] | [--cell CELL]" >&2 exit 2 fi @@ -25,7 +30,7 @@ cleanup() { } trap cleanup EXIT -bin/model-seat-inventory.sh --no-header > "$inventory" +bin/model-agent-inventory.sh --no-header > "$inventory" hydrate_gitdirs() { local root="$1" gitdir target @@ -45,12 +50,20 @@ hydrate_gitdirs() { prepare_cell() { local cell="$1" root="$2" fixture="cells/$cell/fixture" script + case "$cell" in + agent-new-interview-hard-constraint|agent-new-interview-investigation|agent-new-interview-small-fix) + fixture="cells/agent-new-interview/fixture" + ;; + esac [ -d "$fixture" ] || { echo "FAIL: model-backed cell $cell has no fixture" >&2 return 1 } cp -a "$fixture"/. "$root"/ case "$cell" in + agent-new-interview|agent-new-interview-hard-constraint|agent-new-interview-investigation|agent-new-interview-small-fix) + script="$root/prepare-interviewer-worktree.sh" + ;; signal-rename|signal-rename-codex) script="$root/materialize.sh" ;; @@ -73,13 +86,20 @@ prepare_cell() { echo "FAIL: $cell materializer contains a provider/reconcile/network command" >&2 return 1 fi - CATALOG="$root" bash "$script" >/dev/null + if [[ "$cell" == agent-new-interview* ]]; then + CATALOG="$root" bash "$script" "$root/interviewer" >/dev/null + else + CATALOG="$root" bash "$script" >/dev/null + fi } declare -A roots=() failed=0 checked=0 -while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do +while IFS=$'\t' read -r cell agent harness workspace st_agent source_kind source_path source_line; do + if [ -n "$selected_cell" ] && [ "$cell" != "$selected_cell" ]; then + continue + fi if [ "$selected" != "all" ] && [ "$harness" != "$selected" ]; then continue fi @@ -92,7 +112,11 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do } fi - relative="${workspace#./}" + if [[ "$workspace" == "\$CATALOG/"* ]]; then + relative="${workspace#\$CATALOG/}" + else + relative="${workspace#./}" + fi target="${roots[$cell]}/$relative" [ -n "$relative" ] || target="${roots[$cell]}" if [ ! -d "$target" ]; then @@ -101,8 +125,28 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do continue fi - kdl="cells/$cell/$cell.kdl" - command_text="$(sed -n "${command_line}p" "$kdl")" + command_text="$(sed -n "${source_line}p" "$source_path")" + axe_launch=0 + if [ "$source_kind" = "canonical-template" ]; then + publisher="$(dirname "$source_path")/publish-interviewer.sh" + bin/check-canonical-agent-template.sh "$source_path" "$publisher" >/dev/null || { + failed=1 + continue + } + [ -s "$(dirname "$source_path")/_templates/bus.st2.md" ] || + { echo "FAIL: $cell/$agent canonical bus overlay source is missing" >&2; failed=1; } + git -C "$target" rev-parse --is-inside-work-tree | grep -Fxq true || + { echo "FAIL: $cell/$agent canonical workspace is not a Git worktree" >&2; failed=1; } + axe_launch=1 + elif grep -Fq 'exec axe agent launch ' <<< "$command_text"; then + axe_launch=1 + grep -Fq -- '--mode managed-unattended' <<< "$command_text" || + { echo "FAIL: $cell/$agent Axe launch omits managed-unattended mode" >&2; failed=1; } + grep -Fq -- '--boot managed-v1' <<< "$command_text" || + { echo "FAIL: $cell/$agent Axe launch omits managed-v1 boot contract" >&2; failed=1; } + ! grep -Fq -- '--account ' <<< "$command_text" || + { echo "FAIL: $cell/$agent durably pins an account instead of using Axe selection" >&2; failed=1; } + fi if [ "$harness" = "Claude" ]; then [ -s "$target/CLAUDE.md" ] || { echo "FAIL: $cell/$agent has no non-empty CLAUDE.md" >&2; failed=1; continue; } @@ -112,8 +156,10 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do { echo "FAIL: $cell/$agent has no non-empty PERSONA.md" >&2; failed=1; } cmp -s harness/claude-settings.local.json "$target/.claude/settings.local.json" || { echo "FAIL: $cell/$agent does not materialize the canonical Claude hooks" >&2; failed=1; } - grep -Fq 'Read CLAUDE.md.' <<< "$command_text" || - { echo "FAIL: $cell/$agent launch does not use its Claude loader" >&2; failed=1; } + if [ "$axe_launch" -eq 0 ]; then + grep -Fq 'Read CLAUDE.md.' <<< "$command_text" || + { echo "FAIL: $cell/$agent launch does not use its Claude loader" >&2; failed=1; } + fi [ ! -e "$target/.codex/hooks.json" ] || { echo "FAIL: $cell/$agent Claude workspace mixes in Codex hooks" >&2; failed=1; } else @@ -121,10 +167,12 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do { echo "FAIL: $cell/$agent has no non-empty AGENTS.md" >&2; failed=1; continue; } cmp -s harness/codex-hooks.json "$target/.codex/hooks.json" || { echo "FAIL: $cell/$agent does not materialize the canonical Codex hooks" >&2; failed=1; } - grep -Fq 'Read AGENTS.md.' <<< "$command_text" || - { echo "FAIL: $cell/$agent launch does not use its Codex loader" >&2; failed=1; } - grep -Fq -- '--dangerously-bypass-hook-trust' <<< "$command_text" || - { echo "FAIL: $cell/$agent launch does not trust the declared Codex hooks" >&2; failed=1; } + if [ "$axe_launch" -eq 0 ]; then + grep -Fq 'Read AGENTS.md.' <<< "$command_text" || + { echo "FAIL: $cell/$agent launch does not use its Codex loader" >&2; failed=1; } + grep -Fq -- '--dangerously-bypass-hook-trust' <<< "$command_text" || + { echo "FAIL: $cell/$agent launch does not trust the declared Codex hooks" >&2; failed=1; } + fi [ ! -e "$target/.claude/settings.local.json" ] || { echo "FAIL: $cell/$agent Codex workspace mixes in Claude hooks" >&2; failed=1; } fi @@ -139,7 +187,7 @@ while IFS=$'\t' read -r cell agent harness workspace st_agent command_line; do done < "$inventory" [ "$checked" -gt 0 ] || { - echo "FAIL: no $selected model seats were checked" >&2 + echo "FAIL: no $selected model agents were checked" >&2 exit 1 } @@ -166,7 +214,7 @@ grep -Fxq $'docs\tjudge:cold-reader\tone-shot offline Claude print grader; no bu } if [ "$failed" -eq 0 ]; then - printf 'PASS: %d %s model seats materialize and use the canonical harness overlay and hooks\n' \ + printf 'PASS: %d %s model agents materialize and use the canonical harness overlay and hooks\n' \ "$checked" "$selected" printf 'PASS: %d derived model-free cells have explicit harness-hook exclusions\n' \ "${#expected[@]}" diff --git a/bin/check-model-policy-mutations.sh b/bin/check-model-policy-mutations.sh index 3ff9a7da..c97f636e 100755 --- a/bin/check-model-policy-mutations.sh +++ b/bin/check-model-policy-mutations.sh @@ -59,3 +59,53 @@ grep -Fq 'selects Opus' "$opus_output" || { } echo "PASS: nested provider launches are counted independently; unpinned and Opus mutations fail" + +mkdir -p "$scratch/axe-valid" "$scratch/axe-pinned" +printf '%s\n' \ + 'team "x" { agent "i" { workspace "./i"; env { ST_AGENT "x.i" }; command #"exec axe agent launch --harness claude --persona generalist --model claude-sonnet-5 --effort medium --mode managed-unattended --boot managed-v1"#; ding } }' \ + >"$scratch/axe-valid/axe.kdl" +printf '%s\n' \ + 'team "x" { agent "i" { workspace "./i"; env { ST_AGENT "x.i" }; command #"exec axe agent launch --harness claude --persona generalist --model claude-sonnet-5 --effort medium --mode managed-unattended --boot managed-v1 --account claude/example"#; ding } }' \ + >"$scratch/axe-pinned/axe.kdl" + +bash "$checker" "$scratch/axe-valid" >/dev/null || { + echo "FAIL: model policy rejected a fully typed, account-neutral Axe launch" >&2 + exit 1 +} +axe_pinned_output="$scratch/axe-pinned.out" +if bash "$checker" "$scratch/axe-pinned" >"$axe_pinned_output" 2>&1; then + echo "FAIL: model policy accepted a durable Axe account pin" >&2 + exit 1 +fi +grep -Fq 'durably pins an account' "$axe_pinned_output" || { + echo "FAIL: pinned Axe account mutation lacked the expected diagnostic" >&2 + cat "$axe_pinned_output" >&2 + exit 1 +} + +echo "PASS: typed Axe launches are counted and durable account pins fail" + +mkdir -p "$scratch/canonical-valid" "$scratch/canonical-pinned" +printf '%s\n' \ + 'agent "i" { argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--persona" "generalist" "--model" "claude-sonnet-5" "--effort" "medium" "--mode" "managed-unattended" "--boot" "managed-v1" }' \ + >"$scratch/canonical-valid/agent.kdl.template" +printf '%s\n' \ + 'agent "i" { argv "@ADAPTER@" "agent" "launch" "--harness" "claude" "--persona" "generalist" "--model" "claude-sonnet-5" "--effort" "medium" "--mode" "managed-unattended" "--boot" "managed-v1" "--account" "claude/example" }' \ + >"$scratch/canonical-pinned/agent.kdl.template" + +bash "$checker" "$scratch/canonical-valid" >/dev/null || { + echo "FAIL: model policy rejected a canonical managed-launch template" >&2 + exit 1 +} +canonical_pinned_output="$scratch/canonical-pinned.out" +if bash "$checker" "$scratch/canonical-pinned" >"$canonical_pinned_output" 2>&1; then + echo "FAIL: model policy accepted an account pin in a canonical managed-launch template" >&2 + exit 1 +fi +grep -Fq 'durably pins an account' "$canonical_pinned_output" || { + echo "FAIL: canonical template account pin lacked the expected diagnostic" >&2 + cat "$canonical_pinned_output" >&2 + exit 1 +} + +echo "PASS: canonical managed-launch templates are counted and durable account pins fail" diff --git a/bin/check-model-policy.sh b/bin/check-model-policy.sh index 52133e6e..f2b53ed0 100755 --- a/bin/check-model-policy.sh +++ b/bin/check-model-policy.sh @@ -29,8 +29,14 @@ while IFS=: read -r file line text; do if [[ "$file" == *.kdl ]]; then code="${code%%//*}" fi + if [[ "$file" == *.kdl.template && "$code" == *argv* && "$code" == *'"agent" "launch"'* ]]; then + code="${code//\"/}" + code="${code/agent launch/axe agent launch}" + fi remaining="$code" + remaining="${remaining//--harness claude/--harness claude_harness}" + remaining="${remaining//--harness codex/--harness codex_harness}" provider_regex='(^|[^[:alnum:]_-])(exec[[:space:]]+)?(claude|codex)[[:space:]]+-' while [[ "$remaining" =~ $provider_regex ]]; do match="${BASH_REMATCH[0]}" @@ -65,10 +71,55 @@ while IFS=: read -r file line text; do remaining="$after" done + + axe_remaining="$code" + axe_regex='(^|[^[:alnum:]_-])(exec[[:space:]]+)?axe[[:space:]]+agent[[:space:]]+launch[[:space:]]+' + while [[ "$axe_remaining" =~ $axe_regex ]]; do + axe_match="${BASH_REMATCH[0]}" + axe_after="${axe_remaining#*"$axe_match"}" + axe_invocation="$axe_match$axe_after" + if [[ "$axe_after" =~ $axe_regex ]]; then + next_axe_match="${BASH_REMATCH[0]}" + axe_invocation="$axe_match${axe_after%%"$next_axe_match"*}" + fi + + ((launches += 1)) + [[ "$axe_invocation" != *"--account "* ]] || + fail "$file:$line Axe launch durably pins an account instead of selecting per run" + [[ "$axe_invocation" == *"--persona "* ]] || + fail "$file:$line Axe launch omits --persona" + [[ "$axe_invocation" == *"--mode managed-unattended"* ]] || + fail "$file:$line Axe launch omits --mode managed-unattended" + [[ "$axe_invocation" == *"--boot managed-v1"* ]] || + fail "$file:$line Axe launch omits --boot managed-v1" + + if [[ "$axe_invocation" == *"--harness claude"* ]]; then + ((claude_launches += 1)) + [[ "$axe_invocation" == *"--model claude-sonnet-5"* ]] || + fail "$file:$line launches Claude through Axe without --model claude-sonnet-5" + [[ "$axe_invocation" == *"--effort medium"* ]] || + fail "$file:$line launches Claude through Axe without --effort medium" + elif [[ "$axe_invocation" == *"--harness codex"* ]]; then + ((codex_launches += 1)) + [[ "$axe_invocation" == *"--model gpt-5.6-sol"* ]] || + fail "$file:$line launches Codex through Axe without --model gpt-5.6-sol" + [[ "$axe_invocation" == *"--effort medium"* ]] || + fail "$file:$line launches Codex through Axe without --effort medium" + else + fail "$file:$line Axe launch has no supported explicit --harness claude|codex" + fi + + axe_remaining="$axe_after" + done done < <( - rg --no-ignore -n --no-heading \ - 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-' \ - "$scan_root" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true + { + rg --no-ignore -n --no-heading \ + 'exec[[:space:]]+(claude|codex)|(^|[^[:alnum:]_-])(claude|codex)[[:space:]]+-|axe[[:space:]]+agent[[:space:]]+launch' \ + "$scan_root" -g '*.kdl' -g '*.sh' -g '!**/_git/**' || true + rg --no-ignore -n --no-heading \ + '"agent"[[:space:]]+"launch"' \ + "$scan_root" -g '*.kdl.template' || true + } ) [ "$launches" -gt 0 ] || fail "no maintained Claude or Codex launch sites were found" diff --git a/bin/check-no-pii-history.sh b/bin/check-no-pii-history.sh index ea165a90..75a9c051 100755 --- a/bin/check-no-pii-history.sh +++ b/bin/check-no-pii-history.sh @@ -2,6 +2,9 @@ # Mutation-check check-no-pii.sh against a path committed and then deleted from a published _git fixture. set -euo pipefail +export GIT_CONFIG_GLOBAL=/dev/null +export GIT_CONFIG_NOSYSTEM=1 + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" checker="${1:-$repo_root/bin/check-no-pii.sh}" scratch="$(mktemp -d)" diff --git a/bin/check-preflight-safety.sh b/bin/check-preflight-safety.sh index 4af8f41e..a2770e80 100755 --- a/bin/check-preflight-safety.sh +++ b/bin/check-preflight-safety.sh @@ -7,6 +7,13 @@ cd "$repo_root" reachable=( bin/check-corpus.sh + bin/check-append-only-file.sh + bin/check-agent-new-behavior-cases.sh + bin/check-agent-new-interview-attempts-mutations.sh + bin/check-agent-new-renderer-security.sh + bin/check-agent-new-interview-attempts.sh + bin/check-canonical-agent-template-mutations.sh + bin/check-canonical-agent-template.sh bin/check-event-first.sh bin/check-fixture-reset.sh bin/check-fixture-reset-terminal.sh @@ -20,15 +27,23 @@ reachable=( bin/check-preflight-safety.sh bin/check-run-history.sh bin/check-retired-surfaces.sh + bin/check-st2-package-provenance-mutations.sh + bin/check-st2-package-provenance.sh + bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh bin/check-weird-git-setup.sh bin/corpus-inventory.sh bin/generate-catalog.sh - bin/model-seat-inventory.sh + bin/model-agent-inventory.sh + bin/st2-pin.sh ) materializers=( + cells/agent-new-interview/fixture/prepare-interviewer-worktree.sh + cells/agent-new-bundle-contract/fixture/render-intent.sh + cells/agent-new-interview/fixture/interviewer/render-intent.sh + cells/canonical-agent-runtime-smoke/fixture/prepare-interviewer-worktree.sh cells/signal-rename/fixture/materialize.sh cells/signal-rename-codex/fixture/materialize.sh cells/weird-git-setup/fixture/setup-megarepo.sh @@ -56,6 +71,11 @@ mapfile -t direct < <( LC_ALL=C sort -u ) expected_direct=( + bin/check-agent-new-behavior-cases.sh + bin/check-agent-new-interview-attempts-mutations.sh + bin/check-agent-new-interview-attempts.sh + bin/check-agent-new-renderer-security.sh + bin/check-canonical-agent-template-mutations.sh bin/check-event-first.sh bin/check-fixture-reset-terminal.sh bin/check-harness-contract.sh @@ -68,12 +88,15 @@ expected_direct=( bin/check-preflight-safety.sh bin/check-retired-surfaces.sh bin/check-run-history.sh + bin/check-st2-package-provenance-mutations.sh + bin/check-st2-package-provenance.sh + bin/check-st2-pin-consistency.sh bin/check-st2-semantic.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh bin/check-weird-git-setup.sh bin/generate-catalog.sh - bin/model-seat-inventory.sh + bin/model-agent-inventory.sh ) if [ "${direct[*]}" != "${expected_direct[*]}" ]; then fail "check-corpus.sh direct command set differs from the reviewed allowlist" @@ -85,7 +108,6 @@ for file in "${reachable[@]}"; do while IFS= read -r hit; do line="${hit#*:}" case "$line" in - *'st2 --version'*) ;; *'st2 ls '*) ;; *'st2 validate '*) ;; *'st2 up '*'--materialize-only'*) ;; diff --git a/bin/check-st2-package-provenance-mutations.sh b/bin/check-st2-package-provenance-mutations.sh new file mode 100755 index 00000000..0c53549f --- /dev/null +++ b/bin/check-st2-package-provenance-mutations.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +source bin/st2-pin.sh + +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +bin/check-st2-package-provenance.sh >/dev/null + +mkdir -p "$scratch/other-st2-checkout/bin" +substitute="$scratch/other-st2-checkout/bin/st2" +cat >"$substitute" <