diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..bdcbb6ad --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# A unified diff legitimately carries a bare space on blank context lines; the corpus whitespace gate must +# not read the reviewed proposal fixture as an authoring defect. +cells/**/review/*.patch -whitespace diff --git a/CATALOG.md b/CATALOG.md index a4f12707..b01e9267 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,8 +1,8 @@ # Current eval catalog and evidence -This is the sole current corpus surface: **59 included cells** (19 model-free, -40 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active +This is the sole current corpus surface: **71 included cells** (19 model-free, +52 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active KDL and executable judge scripts. Latest accepted PASS evidence remains distinct from the append-only run history: a failed last run is visible without being advertised as accepted evidence, and cells with no structured run row say so explicitly. @@ -29,6 +29,18 @@ harness-native loader plus canonical hook file. | `assignment-contract-hot-resources` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `context-resource-continuity` | model-free | — | 0 | none | `90s` | 4 | — | **NO STRUCTURED RUN** | | `crash-ding` | mixed | `claude-sonnet-5+gpt-5.6-sol` / medium | 4 | high | `180s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-implement-claude-native` | Claude | `claude-sonnet-5` / medium | 1 | low | `900s` | 7 | — | **NO STRUCTURED RUN** | +| `delegation-implement-claude-st2` | Claude | `claude-sonnet-5` / medium | 2 | medium | `900s` | 7 | — | **NO STRUCTURED RUN** | +| `delegation-implement-codex-native` | Codex | `gpt-5.6-sol` / medium | 1 | low | `900s` | 7 | — | **NO STRUCTURED RUN** | +| `delegation-implement-codex-st2` | Codex | `gpt-5.6-sol` / medium | 2 | medium | `900s` | 7 | — | **NO STRUCTURED RUN** | +| `delegation-review-claude-native` | Claude | `claude-sonnet-5` / medium | 1 | low | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-review-claude-st2` | Claude | `claude-sonnet-5` / medium | 3 | high | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-review-codex-native` | Codex | `gpt-5.6-sol` / medium | 1 | low | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-review-codex-st2` | Codex | `gpt-5.6-sol` / medium | 3 | high | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-sweep-claude-native` | Claude | `claude-sonnet-5` / medium | 1 | low | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-sweep-claude-st2` | Claude | `claude-sonnet-5` / medium | 3 | high | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-sweep-codex-native` | Codex | `gpt-5.6-sol` / medium | 1 | low | `900s` | 5 | — | **NO STRUCTURED RUN** | +| `delegation-sweep-codex-st2` | Codex | `gpt-5.6-sol` / medium | 3 | high | `900s` | 5 | — | **NO STRUCTURED RUN** | | `ding-mode` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | | `ding-reply` | Claude | `claude-sonnet-5` / medium | 1 | low | `420s` | 2 | — | **NO STRUCTURED RUN** | | `docs` | Claude | `claude-sonnet-5` / medium | 3 | high | `1200s` | 5 | — | **NO STRUCTURED RUN** | diff --git a/README.md b/README.md index 29f7d5ef..43a49c78 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,41 @@ and include model-free regression tests for those oracle boundaries. These pre-p evidence, not accepted corpus receipts; `CATALOG.md` remains authoritative and will show no accepted PASS until a committed cell is rerun. +## Delegation-parity tournament + +The twelve `delegation-*` cells form one matched tournament asking whether **st2-managed sub-agents are worse +than harness-native sub-agents** on real delegation work. It is a regression check on an existing fleet +decision, not a benchmark: the preregistered question, metrics, replication plan, falsification rule, and +limitations live in +[`evidence/delegation-parity-design-20260812.md`](evidence/delegation-parity-design-20260812.md). + +| Task | `claude-st2` / `codex-st2` | `claude-native` | `codex-native` | +| --- | --- | --- | --- | +| Broad multi-file search (`sweep`, fan-out 2) | peer seats over the native bus | one seat, `Agent` sub-agents | one seat, `spawn_agent` sub-agents | +| Independent review of a diff (`review`, fan-out 2) | peer seats over the native bus | one seat, `Agent` sub-agents | one seat, `spawn_agent` sub-agents | +| Scoped implementation (`implement`, fan-out 1) | peer seats over the native bus | one seat, `Agent` sub-agents | one seat, `spawn_agent` sub-agents | + +Within one task the four arms are byte-identical in task, held-out graders, frozen product repository, +deliverable contract, and timeout; only the delegation layer differs, and the outcome graders never learn the +arm. Each native arm has a same-harness managed counterpart, because the decision-relevant comparison is +within a harness family. Deliverables land in a shared `findings/` directory with per-delegate attribution, so +one mechanical grader reads a bus delegation and a native fan-out identically. + +Two honest boundaries: the native arms necessarily run **unmanaged** — a folder eval launches the harness +directly, so the fleet's native-spawn refusal never fires — and native fan-out is **self-attested**. The only +mechanical guarantee there is negative: a native cell declares exactly one bus seat and its delegation judge +fails if a second mailbox ever appears. Timing is measured as bus latency between the kickoff receipt and the +coordinator's confirmation, because a single-seat folder eval never signals completion and therefore always +consumes its whole `max-timeout`. + +Matching, grader discrimination, and the tasks' premises are proven for free, without starting a model: + +```sh +bin/check-delegation-parity.sh +``` + +No arm has been run. `CATALOG.md` will show no accepted PASS for these cells until one is. + ## Add or change a cell 1. Create exactly one `cells//.kdl`. diff --git a/bin/check-corpus.sh b/bin/check-corpus.sh index b3a66ac1..e2404314 100755 --- a/bin/check-corpus.sh +++ b/bin/check-corpus.sh @@ -40,6 +40,7 @@ bin/check-fixture-reset-terminal.sh bin/check-harness-contract.sh bin/check-vrs-scope-drift.sh bin/check-vrs-variations.sh +bin/check-delegation-parity.sh bin/check-weird-git-setup.sh bin/check-preflight-closed-set-mutations.sh bin/check-overnight-policy.sh diff --git a/bin/check-delegation-parity.sh b/bin/check-delegation-parity.sh new file mode 100755 index 00000000..5e91da9f --- /dev/null +++ b/bin/check-delegation-parity.sh @@ -0,0 +1,522 @@ +#!/usr/bin/env bash +# Model-free matching and oracle-validity gate for the 12-cell delegation-parity tournament. +# +# It proves two things without starting a model: +# 1. MATCHING — within one task, the four arms differ only in the delegation layer. Task bytes, held-out +# graders, the frozen baseline repository, the deliverable contract, and the timeout are identical, and +# the graders and mutation inputs are never visible inside a fixture. +# 2. ORACLE VALIDITY — a complete correct outcome passes every gating judge in both the st2 and the native +# bus shape, and each preregistered planted failure is rejected by the judge that is supposed to catch it. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +tasks=(sweep review implement) +arms=(claude-st2 claude-native codex-st2 codex-native) + +scratch="$(mktemp -d)" +cleanup() { + rm -rf -- "$scratch" +} +trap cleanup EXIT + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +# ── 1. matching ──────────────────────────────────────────────────────────────────────────────────────────── +reference_arm=claude-st2 +for task in "${tasks[@]}"; do + ref="cells/delegation-$task-$reference_arm" + [ -d "$ref" ] || fail "delegation-$task-$reference_arm is missing" + for arm in "${arms[@]}"; do + cell="cells/delegation-$task-$arm" + [ -d "$cell" ] || fail "delegation-$task-$arm is missing" + + cmp -s "$ref/task.md" "$cell/task.md" || + fail "delegation-$task-$arm has a different frozen task than $reference_arm" + cmp -s "$ref/judges/grade.sh" "$cell/judges/grade.sh" || + fail "delegation-$task-$arm has a different outcome grader than $reference_arm" + cmp -s "$ref/judges/observe.sh" "$cell/judges/observe.sh" || + fail "delegation-$task-$arm has a different observation recorder than $reference_arm" + cmp -s "$ref/judges/repo.sha256" "$cell/judges/repo.sha256" || + fail "delegation-$task-$arm has a different frozen baseline manifest than $reference_arm" + cmp -s "$ref/fixture/findings/CONTRACT.md" "$cell/fixture/findings/CONTRACT.md" || + fail "delegation-$task-$arm has a different deliverable contract than $reference_arm" + diff -r "$ref/fixture/repo" "$cell/fixture/repo" >/dev/null || + fail "delegation-$task-$arm has a different product repository than $reference_arm" + if [ -d "$ref/mutations" ]; then + diff -r "$ref/mutations" "$cell/mutations" >/dev/null || + fail "delegation-$task-$arm has different held-out mutation inputs than $reference_arm" + fi + if [ -d "$ref/fixture/review" ]; then + diff -r "$ref/fixture/review" "$cell/fixture/review" >/dev/null || + fail "delegation-$task-$arm has a different proposal under review than $reference_arm" + fi + + ref_timeout="$(grep -E '^[[:space:]]*max-timeout ' "$ref/delegation-$task-$reference_arm.kdl")" + arm_timeout="$(grep -E '^[[:space:]]*max-timeout ' "$cell/delegation-$task-$arm.kdl")" + [ "$ref_timeout" = "$arm_timeout" ] || + fail "delegation-$task-$arm declares $arm_timeout, not the matched $ref_timeout" + + ref_kickoff="$(grep -E '^[[:space:]]*to "' "$ref/delegation-$task-$reference_arm.kdl")" + arm_kickoff="$(grep -E '^[[:space:]]*to "' "$cell/delegation-$task-$arm.kdl")" + [ "$ref_kickoff" = "$arm_kickoff" ] || + fail "delegation-$task-$arm delivers the kickoff to $arm_kickoff, not the matched $ref_kickoff" + + # Held-out assets must never be reachable from an agent's world. + if find "$cell/fixture" -type f \ + \( -name 'grade.sh' -o -name 'observe.sh' -o -name 'repo.sha256' -o -name '*-slug.js' -o -name '*-test.js' \) | + grep -q .; then + fail "delegation-$task-$arm leaks a held-out grader or mutation input into its fixture" + fi + + seats="$(grep -cE '^ agent "' "$cell/delegation-$task-$arm.kdl")" + case "$arm" in + *-native) + [ "$seats" -eq 1 ] || + fail "delegation-$task-$arm is a native arm with $seats bus seats; a native arm delegates without peers" + grep -q 'grade.sh delegation native' "$cell/delegation-$task-$arm.kdl" || + fail "delegation-$task-$arm does not grade native delegation evidence" + ;; + *-st2) + [ "$seats" -ge 2 ] || + fail "delegation-$task-$arm is an st2 arm with $seats bus seat(s); it needs a coordinator and delegates" + grep -q 'grade.sh delegation st2' "$cell/delegation-$task-$arm.kdl" || + fail "delegation-$task-$arm does not grade bus delegation evidence" + ;; + esac + done + + # The frozen manifest must describe the shipped fixture exactly. + ( cd "$ref/fixture" && sha256sum -c --quiet "$repo_root/$ref/judges/repo.sha256" >/dev/null 2>&1 ) || + fail "delegation-$task baseline manifest does not match the shipped fixture repository" +done + +# The shared delegation layer is itself matched: one coordinator persona per arm kind, one delegate persona. +for task in "${tasks[@]}"; do + cmp -s cells/delegation-sweep-claude-st2/fixture/sup/PERSONA.md \ + "cells/delegation-$task-claude-st2/fixture/sup/PERSONA.md" || + fail "the st2 coordinator persona differs between tasks" + cmp -s cells/delegation-sweep-claude-st2/fixture/sup/PERSONA.md \ + "cells/delegation-$task-codex-st2/fixture/sup/AGENTS.md" || + fail "the st2 coordinator persona differs between harnesses" + cmp -s cells/delegation-sweep-claude-st2/fixture/w1/PERSONA.md \ + "cells/delegation-$task-claude-st2/fixture/w1/PERSONA.md" || + fail "the st2 delegate persona differs between tasks" + cmp -s cells/delegation-sweep-claude-native/fixture/sup/PERSONA.md \ + "cells/delegation-$task-claude-native/fixture/sup/PERSONA.md" || + fail "the Claude native coordinator persona differs between tasks" + cmp -s cells/delegation-sweep-codex-native/fixture/sup/AGENTS.md \ + "cells/delegation-$task-codex-native/fixture/sup/AGENTS.md" || + fail "the codex native coordinator persona differs between tasks" +done + +# The st2 coordinator can only delegate to identities it can name: a hermetic eval catalog holds no Agent Spec +# declarations on disk, so roster discovery there reports the fixture's own files instead of the peer seats. +# Every declared delegate identity must therefore appear verbatim in the shared coordinator persona. +sup_persona=cells/delegation-sweep-claude-st2/fixture/sup/PERSONA.md +for task in "${tasks[@]}"; do + kdl="cells/delegation-$task-claude-st2/delegation-$task-claude-st2.kdl" + team="$(sed -n 's/^team "\([^"]*\)".*/\1/p' "$kdl" | head -1)" + [ -n "$team" ] || fail "delegation-$task-claude-st2 declares no team prefix" + while IFS= read -r seat; do + [ "$seat" = sup ] && continue + grep -Fq "\`$team.$seat\`" "$sup_persona" || + fail "the st2 coordinator persona never names the declared delegate $team.$seat" + done < <(sed -n 's/^ agent "\([^"]*\)".*/\1/p' "$kdl") +done + +# The deliverable contract is arm-neutral, but it must not contradict a persona that requires a delegation log. +for task in "${tasks[@]}"; do + grep -Fq 'delegation-log.md' "cells/delegation-$task-claude-st2/fixture/findings/CONTRACT.md" || + fail "the delegation-$task deliverable contract does not admit the native delegation log it is graded on" +done + +# Each native persona must actually name its harness spawn surface, and no st2 persona may name one. +grep -q '`Agent` tool' cells/delegation-sweep-claude-native/fixture/sup/PERSONA.md || + fail "the Claude native persona does not name the Agent tool" +grep -q 'spawn_agent' cells/delegation-sweep-codex-native/fixture/sup/AGENTS.md || + fail "the codex native persona does not name spawn_agent" +if grep -qE 'Agent. tool|spawn_agent' cells/delegation-sweep-claude-st2/fixture/sup/PERSONA.md; then + fail "the st2 coordinator persona names a harness-native spawn tool" +fi + +printf 'PASS: %d delegation-parity cells match within each task and hold their graders out\n' \ + "$(( ${#tasks[@]} * ${#arms[@]} ))" + +# ── 2. the baseline world behaves as the frozen tasks claim ──────────────────────────────────────────────── +baseline="$scratch/baseline" +cp -a cells/delegation-sweep-claude-st2/fixture/repo "$baseline" +( cd "$baseline" && node --test >/dev/null 2>&1 ) || + fail "the baseline product repository does not start with a green suite" + +patched="$scratch/patched" +cp -a "$baseline" "$patched" +( cd "$patched" && git apply "$repo_root/cells/delegation-review-claude-st2/fixture/review/proposed.patch" ) || + fail "the reviewed proposal does not apply to the frozen baseline" +( cd "$patched" && node --test >/dev/null 2>&1 ) || + fail "the reviewed proposal is not green, so the review task's premise is false" + +cmp -s cells/delegation-implement-claude-st2/mutations/baseline-slug.js \ + cells/delegation-implement-claude-st2/fixture/repo/src/slug.js || + fail "the held-out pre-change implementation is not the one shipped in the fixture" + +printf 'PASS: the baseline suite is green, the reviewed proposal applies and stays green, and the frozen pre-change implementation matches\n' + +# ── 3. oracle validity ──────────────────────────────────────────────────────────────────────────────────── +counter=0 +put_msg() { # put_msg + counter=$((counter + 1)) + mkdir -p "$1" + printf 'from: %s\nto: peer\n\nsynthetic oracle message\n' "$3" > "$1/$2-o${counter}aa.md" +} + +team_of() { # every cell uses one uniform team prefix, which is what lets one persona name its delegates + printf 'dg\n' +} + +workers_of() { + case "$1" in + implement) printf 'w1\n' ;; + *) printf 'w1\nw2\n' ;; + esac +} + +build_catalog() { # build_catalog -> catalog path + local name="$1" task="$2" shape="$3" + local root="$scratch/$name" team worker + team="$(team_of "$task")" + rm -rf -- "$root" + mkdir -p "$root" "$root/requester/inbox" "$root/requester/archive" \ + "$root/evalhost.$team.sup/inbox" "$root/evalhost.$team.sup/archive" + cp -a "cells/delegation-$task-claude-st2/fixture/." "$root/" + put_msg "$root/evalhost.$team.sup/archive" 1780000000000 requester + if [ "$shape" = st2 ]; then + while IFS= read -r worker; do + mkdir -p "$root/evalhost.$team.$worker/inbox" "$root/evalhost.$team.$worker/archive" + put_msg "$root/evalhost.$team.$worker/archive" 1780000010000 "evalhost.$team.sup" + put_msg "$root/evalhost.$team.sup/archive" 1780000020000 "evalhost.$team.$worker" + done < <(workers_of "$task") + fi + put_msg "$root/requester/inbox" 1780000030000 "evalhost.$team.sup" + printf '%s\n' "$root" +} + +judge() { # judge [arm] + local task="$1" catalog="$2" + shift 2 + CATALOG="$catalog" ST_ROOT="$catalog" SPEC_DIR="$repo_root/cells/delegation-$task-claude-st2" \ + bash "cells/delegation-$task-claude-st2/judges/grade.sh" "$@" +} + +expect_pass() { # expect_pass