fix(automationbench): record the harness's exit status on the k8s surface too

.agents/benchmarks/RULES.md

@@ -186,3 +186,4 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S
 | 2026-08-11 | New rule **5a**: rule 5 bounds what a benchmark may expose, not what it must withhold — where the upstream benchmark's definition includes returning something to the solver, reproducing it is correctness. Evaluative *output* is not evaluation code, so a benchmark whose upstream metric is defined after a round of test feedback may return test output to the agent; the test files, grading logic and expected answers stay unreadable and unmodifiable. Prompted by aider-polyglot, whose leaderboard metric (`pass_rate_2`) is measured after exactly one such round — reporting the first attempt against it understated the score by ~30 points. |
 | 2026-09-02 | Rule **24f**: the chart resolves `perTask` itself, from a committed `<chart>/per-task.json` derived from the per-task labels (`chart_per_task_set_matches_labels` keeps the two equal) — the same chart-resolves-it-from-its-own-data model rule 24h already sets for sidecars. `run --mode job` derived it from `containers/benchmarks/<b>/Dockerfile` at a **cwd-relative** path while defaulting its own chart source to the published OCI chart "so `--mode job` needs no repo checkout": run from anywhere else, every per-task benchmark rendered the shared-env image name and the Job died in ImagePullBackOff. One resolver now serves every helm caller — CLI, `deploy/oc/`, and the dashboard's vendored chart. `deploy/oc/run.sh` refuses per-task benchmarks outright (the internal registry cannot build them: `build --builder oc` has no `--task-id`, and a flat ImageStream name cannot hold `sympy__sympy-24066`) and `sweep.sh` skips them by name instead of aborting the grid. |
 | 2026-09-06 | New rule **21c**: every benchmark MUST carry both `LABEL eval.benchmark.internet` and `ENV EVAL_INTERNET` with the same value — a label alone leaves the declared policy unenforced. All ~104 benchmarks that previously declared the label without the `ENV` were given the matching `ENV` in the same PR (#423); enforced mechanically by `tests/static/policy/dockerfile/labels.rego`. |
+| 2026-09-10 | Rules **14/16/24b** enforced for presets that replace the agent phase. `presets/automationbench.yaml` ran its own harness with `|| true`, so `run-agent` never ran, nothing wrote `/output/agent/.exit-code`, and `write-result` coerced `agent/result.json` `exit_code` to null — while its `compose.yaml` twin recorded the code, a rule 24b lockstep break that made the two surfaces disagree on a rule-16 field. Downstream, a task killed at the wall clock was indistinguishable from one that never started (the dashboard labelled 30 timed-out tasks `incomplete`, not `timeout`). Both surfaces now bound the harness with `timeout -k 30 $TIMEOUT` (rule 14) and record `$?`; the inner bound MUST fire before the pod's `activeDeadlineSeconds` (`timeout + deadlineGrace`), whose SIGKILL leaves no shell alive to record anything. Rule 29's per-surface checks could not catch this — each surface rendered fine alone and the drift lived in `runnerArgs`, the one per-benchmark command a preset may still define — so two `tests/static/check.rs` gates now hold every `runnerArgs` preset to recording an exit status and bounding its harness. |

containers/benchmarks/_chart/presets/automationbench.yaml

@@ -12,9 +12,19 @@ timeout: '1800'
 # `process_completed` semantics): write-result always grades whatever was
 # produced, fail-closed to 0. run_automationbench.py resolves the task from
 # /tasks/all.jsonl directly, so it does not depend on the materializer.
+#
+# There is no separate agent phase here, so run-agent never runs and nothing
+# writes /output/agent/.exit-code (rule 16) — write-result then coerces it to
+# JSON null, and a task killed at the wall clock is indistinguishable from one
+# that never started. Bound the harness the way run-agent bounds an agent
+# (`timeout -k 30 $TIMEOUT`, rule 14) and record its status, so a timeout lands
+# as 124 rather than as nothing. The inner bound must fire BEFORE the pod's
+# activeDeadlineSeconds (timeout + deadlineGrace), which SIGKILLs the whole pod
+# and leaves no shell alive to record anything.
 runnerArgs: >-
   /eval-materialize-task || true;
-  python3 /app/run_automationbench.py || true;
+  timeout -k 30 "${TIMEOUT:-1800}" python3 /app/run_automationbench.py;
+  printf '%s' "$?" > /output/agent/.exit-code;
   /usr/local/bin/write-result;
   /usr/local/bin/reap-sidecars
 resources:

containers/benchmarks/automationbench/compose.yaml

@@ -20,13 +20,15 @@ services:
     # step runs regardless of the previous one's status — write-result grades
     # whatever was produced, fail-closed to 0 (mirrors /usr/local/bin/run).
     # There is no separate agent phase here, so run-agent never writes
-    # /output/agent/.exit-code (rule 16) — capture the harness's own exit
-    # status into it instead, so agent/result.json's exit_code reflects the
-    # program that actually ran rather than staying permanently null.
+    # /output/agent/.exit-code (rule 16) — bound the harness the way run-agent
+    # bounds an agent (`timeout -k 30 $TIMEOUT`, rule 14) and capture its exit
+    # status into that file, so agent/result.json's exit_code reflects the
+    # program that actually ran rather than staying permanently null, and a
+    # wall-clock kill lands as 124 rather than as nothing.
     entrypoint:
       - bash
       - -c
-      - "/eval-materialize-task || true; python3 /app/run_automationbench.py; echo $? > /output/agent/.exit-code; /usr/local/bin/write-result"
+      - '/eval-materialize-task || true; timeout -k 30 "${TIMEOUT:-1800}" python3 /app/run_automationbench.py; printf %s "$?" > /output/agent/.exit-code; /usr/local/bin/write-result'
     environment:
       BENCHMARK: automationbench
       # The harness talks to the gateway on the OpenAI-compatible surface.

tests/static/check.rs

@@ -1429,3 +1429,82 @@ fn a_dispatch_runs_in_its_own_lane() {
          and evicts other pending dispatches"
     );
 }
+
+/// Rule 16 requires `agent/result.json` to carry an `exit_code`, and rule 24
+/// requires every surface to produce byte-equivalent results for the same
+/// inputs. Only `run-agent` writes `/output/agent/.exit-code`, so a benchmark
+/// whose preset replaces the standard three-phase pipeline with its own harness
+/// (`runnerArgs`) must record the status itself — otherwise `write-result`
+/// coerces it to JSON null and every task of that benchmark reports an
+/// unrecorded exit status for the rest of time.
+///
+/// automationbench shipped exactly that gap on the k8s surface while its compose
+/// twin recorded the code, so the two surfaces disagreed (a rule 24b lockstep
+/// violation) and a wall-clock kill was indistinguishable from a task that never
+/// started. Rule 29's per-surface checks cannot catch it: each surface renders
+/// fine on its own, and the drift lives in the one per-benchmark command a
+/// preset is still allowed to define.
+#[test]
+fn a_preset_that_replaces_the_agent_phase_records_its_exit_status() {
+    let presets = repo_root().join("containers/benchmarks/_chart/presets");
+    let mut checked = 0;
+    for entry in fs::read_dir(&presets).expect("read presets dir") {
+        let path = entry.expect("dir entry").path();
+        if path.extension().and_then(|e| e.to_str()) != Some("yaml") {
+            continue;
+        }
+        let body = fs::read_to_string(&path).expect("read preset");
+        // Only presets that take over the runner command bypass run-agent.
+        if !body.contains("runnerArgs:") {
+            continue;
+        }
+        let name = path.file_stem().unwrap().to_string_lossy().to_string();
+        // Either record the status into the file write-result reads, or hand the
+        // harness's own status back as the container's (tau-bench's `exit $rc`),
+        // which keeps the Job's outcome honest.
+        let records = body.contains("/output/agent/.exit-code") || body.contains("exit $rc");
+        assert!(
+            records,
+            "preset {name}.yaml overrides runnerArgs (so run-agent never writes \
+             /output/agent/.exit-code) but never records an exit status — \
+             write-result will coerce agent/result.json exit_code to null and a \
+             timeout becomes indistinguishable from a task that never ran \
+             (rules 16, 24)"
+        );
+        checked += 1;
+    }
+    assert!(
+        checked > 0,
+        "expected at least one preset defining runnerArgs"
+    );
+    eprintln!("✓ {checked} runnerArgs preset(s) record an exit status (rules 16, 24)");
+}
+
+/// Rule 14: agent execution MUST be bounded by `EVAL_TIMEOUT`. `run-agent`
+/// enforces it with `timeout -k 30 $TIMEOUT`; a preset that runs its own harness
+/// instead of `run-agent` must enforce it too. Without an inner bound the only
+/// limit left is the pod's `activeDeadlineSeconds` (timeout + deadlineGrace),
+/// which SIGKILLs the whole pod — so nothing survives to write `.exit-code`, and
+/// the recorded-status guarantee above silently stops holding at the wall clock.
+#[test]
+fn a_preset_that_replaces_the_agent_phase_bounds_its_harness() {
+    let presets = repo_root().join("containers/benchmarks/_chart/presets");
+    for entry in fs::read_dir(&presets).expect("read presets dir") {
+        let path = entry.expect("dir entry").path();
+        if path.extension().and_then(|e| e.to_str()) != Some("yaml") {
+            continue;
+        }
+        let body = fs::read_to_string(&path).expect("read preset");
+        if !body.contains("/output/agent/.exit-code") {
+            continue; // covered by the sibling test, or defers to run-agent
+        }
+        let name = path.file_stem().unwrap().to_string_lossy().to_string();
+        assert!(
+            body.contains("timeout -k"),
+            "preset {name}.yaml records its own exit status but never bounds the \
+             harness with `timeout -k … $TIMEOUT` (rule 14) — the pod's \
+             activeDeadlineSeconds SIGKILL would then be the only limit, and it \
+             leaves no shell alive to record 124"
+        );
+    }
+}