From 64e391390bc2e525580068d2f261bb7d7fe4ea0e Mon Sep 17 00:00:00 2001 From: Younggi Choi <74581798+choiyounggi@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:12:25 +0900 Subject: [PATCH 1/2] fix(orchestrate): a dead Orca runtime is exit 4, and a timed-out ask is resumed, not decided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two failure modes measured on a live 3-worker run, both of which read as normal operation at the time. 1. orca-wait.sh could not tell a runtime outage from a quiet window. The check result went through `.result.count // 0`, so `{"ok":false,"error":{"code": "runtime_unavailable"}}` (status 1) and a killed CLI (no output) both scored 0 messages and continued as checkpoints until the whole budget was gone, ending on "no message — keep waiting". New exit 4 fires only on an explicit fault: nonzero status, `ok:false`, or `connectionLost:true` mid-wait. A missing `ok` still checkpoints, so a partial payload cannot read as an outage. Envelopes verified against the live CLI: an ordinary timeout is ok:true / count 0 / timedOut:true with status 0; `worker-show` on an unknown dispatch is ok:false with status 1. Exit 4 is deliberately not a restart signal — a dead runtime does not stop a worker session (measured: a worker committed and pushed three times while the runtime was down). 2. The worker prompt told workers to `ask` and end their turn, but said nothing about the window expiring. Orca leaves a timed-out question pending, to be resumed by its original message id; both workers instead proceeded on a self-chosen "conservative assumption" at 600s and 900s and reported the guess afterwards. One happened to match the human decision. Rule [4] now states that a timeout is not an answer, requires `ask --resume `, and forbids re-asking (a second --question creates a second thread). 386/386 bats green. The Orca prompt-set checksum is bumped in this commit, and a new test names the ask-timeout rule so a reword that keeps the command but drops the obligation shows up in that test's diff. Co-Authored-By: Claude Opus 5 --- skills/orchestrate/SKILL.md | 13 ++++- skills/orchestrate/scripts/orca-wait.sh | 29 +++++++++- .../orchestrate/templates/session-prompt.md | 9 ++++ tests/orca-wait.bats | 53 +++++++++++++++++++ tests/send-prompt.bats | 15 +++++- 5 files changed, 113 insertions(+), 6 deletions(-) diff --git a/skills/orchestrate/SKILL.md b/skills/orchestrate/SKILL.md index 5d3aca6..caf20b9 100644 --- a/skills/orchestrate/SKILL.md +++ b/skills/orchestrate/SKILL.md @@ -189,7 +189,12 @@ you instead of making you poll. Replace steps 1–3 below with O1–O5: `GROUNDWORK_ESCALATION_DIR= scripts/orca-wait.sh [--until-all] []` → **0** completions arrived (acked — process them), **2** window elapsed *or* the ack did not land (checkpoint, just re-run), **3** a - worker reported failure, **5** escalation pending (approve/deny, **clear + worker reported failure, **4** the runtime itself did not answer (`ok:false`, a + nonzero status, or a connection lost mid-wait) — an **outage, not a checkpoint**: + run `orca status --json` before waiting again, and restart nothing on this code + alone, because a dead runtime does not stop a worker session (measured: workers + kept committing and pushing while the runtime was down), **5** escalation + pending (approve/deny, **clear `.orchestration/escalations/`**, then re-run — like watch-status, code 5 recurs while a record is still on disk, by design), **6** question pending (`orca orchestration reply --id --body "" --json`, re-run). @@ -237,7 +242,11 @@ durable re-entry state); on top of that it must: --subject "guardrails " --body "" --task-id --dispatch-id --json`; 3. use `orca orchestration ask --question "" --timeout-ms --json` for a - blocking question, and then end its turn. + blocking question, and then end its turn — and if that window expires, resume the + same question with `ask --resume ` rather than deciding it or asking + it again. A timeout leaves the question pending; it is not an answer. Measured on + a 3-worker run: at 600s and 900s both workers instead "proceeded on a conservative + assumption" and reported the guess after the fact. Step 2 is the *fast* path for a guardrails block — it arrives with the worker's own context. It is not the only one: `orca-wait.sh` pre-checks diff --git a/skills/orchestrate/scripts/orca-wait.sh b/skills/orchestrate/scripts/orca-wait.sh index a2ead1c..d42c301 100755 --- a/skills/orchestrate/scripts/orca-wait.sh +++ b/skills/orchestrate/scripts/orca-wait.sh @@ -21,6 +21,11 @@ # (--until-all: some tasks are still running; batches already acked in # this call are consumed and are not replayed) # exit 3 a worker reported outcome=failed (or an unprovable worker_done) +# exit 4 the Orca runtime did not answer the check (ok:false / nonzero status / +# a connection lost mid-wait) — an OUTAGE, not a quiet window. Never +# treat it as a checkpoint: check `orca status --json` first. The workers +# themselves may well still be alive; a dead runtime does not stop a +# worker session, so do not restart anything on this code alone. # exit 5 an escalation is pending — resolve it, then re-run # exit 6 a worker question is pending — `orchestration reply --id `, re-run # @@ -51,6 +56,7 @@ # ORCA_BIN orca executable (default: orca) # ORCA_WAIT_DRYRUN print the ack instead of sending it # ORCA_WAIT_CHECK_JSON canned `check --wait --json` Delivery (tests) +# ORCA_WAIT_CHECK_RC exit status to pair with that canned Delivery (tests) # ORCA_WAIT_ACK_FAIL force the ack to fail (tests) # ORCA_WAIT_TASKLIST_JSON canned `task-list --status completed --json` (tests) set -u @@ -125,13 +131,32 @@ while :; do [ "$slice" -gt "$remaining" ] && slice="$remaining" if [ -n "${ORCA_WAIT_CHECK_JSON:-}" ]; then - out="$ORCA_WAIT_CHECK_JSON" + out="$ORCA_WAIT_CHECK_JSON"; rc="${ORCA_WAIT_CHECK_RC:-0}" else out=$("$ORCA" orchestration check --wait \ - --types worker_done,escalation,question --timeout-ms "$slice" --json 2>/dev/null) || out="" + --types worker_done,escalation,question --timeout-ms "$slice" --json 2>/dev/null); rc=$? fi remaining=$((remaining - slice)) + # A dead runtime is not a quiet window. Measured envelopes: an ordinary timeout + # is `{"ok":true,"result":{"count":0,"timedOut":true,"connectionLost":false}}` + # with status 0, while a runtime fault is `{"ok":false,"error":{"code":...}}` + # with status 1 (`runtime_timeout`, `runtime_unavailable`) and a killed CLI + # answers nothing at all. All three used to fall through `.result.count // 0` + # and read as "no message yet", so an outage burned the whole budget one silent + # checkpoint at a time and then reported "keep waiting" — the coordinator never + # learned the runtime was gone. Only an EXPLICIT fault signal exits 4: a missing + # `ok` stays a checkpoint so a partial or replayed payload can never be reported + # as an outage. + lost=$(printf '%s' "$out" | "$JQ" -r '.result.connectionLost // false' 2>/dev/null) || lost=false + ok=$(printf '%s' "$out" | "$JQ" -r '.ok // true' 2>/dev/null) || ok=true + if [ "$rc" -ne 0 ] || [ "$ok" = false ] || [ "$lost" = true ]; then + ecode=$(printf '%s' "$out" | "$JQ" -r '.error.code // empty' 2>/dev/null) || ecode="" + [ "$lost" = true ] && [ -z "$ecode" ] && ecode="connection_lost" + echo "[orca-wait] orca check failed (${ecode:-no response}, status $rc) — the runtime is not answering, not the workers being quiet; run \`orca status --json\` before waiting again" >&2 + exit 4 + fi + count=$(printf '%s' "$out" | "$JQ" -r '.result.count // 0' 2>/dev/null) || count=0 case "$count" in ''|*[!0-9]*) count=0 ;; esac [ "$count" -lt 1 ] && continue diff --git a/skills/orchestrate/templates/session-prompt.md b/skills/orchestrate/templates/session-prompt.md index 7328f4e..83136c7 100644 --- a/skills/orchestrate/templates/session-prompt.md +++ b/skills/orchestrate/templates/session-prompt.md @@ -161,6 +161,15 @@ and report exactly once: [4] Blocking question — `orca orchestration ask --question "" --timeout-ms --json`, then end your turn. Never open a local interactive prompt: no human is attached to this session, so it blocks until the window expires with nothing to show for it. + **A timeout is not an answer.** The window expiring leaves the question *pending*, + so resume that exact question — `orca orchestration ask --resume + --timeout-ms --json` — and end your turn again. Do NOT decide it yourself, and + do NOT ask it again: a second `--question` creates a second question, and the + coordinator cannot tell which thread it is answering. Measured on a 3-worker run: + at 600s and 900s the workers chose "proceed on a conservative assumption" instead + and reported the guess only afterwards. One guess happened to match the human + decision — that is luck, not a protocol; the other would have cost a rollback. + If you are blocked, stay blocked and resume. [5] `{ORCA_DISPATCH_ID}` is created by `orca-worker-start.sh`, after `task-create`. If the orchestrator left it unsubstituted, use the dispatch id Orca gave you in this diff --git a/tests/orca-wait.bats b/tests/orca-wait.bats index 8fc87e1..bbc1445 100644 --- a/tests/orca-wait.bats +++ b/tests/orca-wait.bats @@ -30,6 +30,7 @@ mk_stub() { # -> path of an executable `orca` test double cat > "$stub" <<'STUBEOF' #!/bin/sh # Test double for the `orca` CLI. All state under $STUB_DIR: +# rc-fail if present, every `check --wait` prints this file and exits 1 # calls window counter for `check --wait` # windows the --timeout-ms value of each window, one per line # .json canned Delivery returned by the nth window (absent -> empty) @@ -52,6 +53,7 @@ case "${2:-}" in printf '{"taskId":"%s","rule":"%s","reason":"blocked"}' \ "${STUB_ESC_TASK:-t_esc}" "${STUB_ESC_RULE:-sql_drop}" > "${STUB_ESC_DIR:?}/esc-$n.json" fi + if [ -f "$d/rc-fail" ]; then cat "$d/rc-fail"; exit 1; fi if [ -f "$d/$n.json" ]; then cat "$d/$n.json" else echo '{"result":{"count":0,"messages":[]}}'; fi ;; --ack) @@ -138,6 +140,57 @@ teardown() { rm -f "$(REPO_TMP)/orca-stub-${BATS_TEST_NUMBER}"; } [ "$status" -eq 2 ] } +# --- a dead runtime is exit 4, never a quiet window ---------------------------- +# Measured against the live CLI: an ordinary timeout answers ok:true / count 0 / +# timedOut:true with status 0, and a runtime fault answers ok:false with status 1. +# Before this split, every one of these read as "no message yet". + +@test "the real timeout envelope is still a checkpoint, not an outage (regression)" { + run env ORCA_WAIT_DRYRUN=1 \ + ORCA_WAIT_CHECK_JSON='{"ok":true,"result":{"runId":"run_1","deliveryId":null,"messages":[],"count":0,"timedOut":true,"connectionLost":false}}' \ + bash "$OW" 1000 + [ "$status" -eq 2 ] + [[ "$output" == *"checkpoint"* ]] +} + +@test "an ok:false runtime error exits 4 and names the error code" { + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_RC=1 \ + ORCA_WAIT_CHECK_JSON='{"ok":false,"error":{"code":"runtime_unavailable","message":"The Orca runtime closed the connection before responding."}}' \ + bash "$OW" 60000 + [ "$status" -eq 4 ] + [[ "$output" == *"runtime_unavailable"* ]] + [[ "$output" != *"checkpoint"* ]] +} + +# An empty ORCA_WAIT_CHECK_JSON cannot express this case — it deselects the canned +# branch — so a killed CLI is only observable through the real orca path. +@test "a killed CLI (no output at all, nonzero status) exits 4, not 2 (error path)" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + : > "$sd/rc-fail" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" 60000 + [ "$status" -eq 4 ] + [[ "$output" == *"no response"* ]] + [ ! -f "$sd/acked" ] +} + +@test "connectionLost mid-wait exits 4 even though ok:true and count 0 (boundary)" { + run env ORCA_WAIT_DRYRUN=1 \ + ORCA_WAIT_CHECK_JSON='{"ok":true,"result":{"count":0,"messages":[],"timedOut":false,"connectionLost":true}}' \ + bash "$OW" 60000 + [ "$status" -eq 4 ] + [[ "$output" == *"connection_lost"* ]] +} + +@test "a nonzero check through the live orca path exits 4 without acking anything" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '{"ok":false,"error":{"code":"runtime_timeout","message":"Timed out waiting for the Orca runtime."}}' > "$sd/rc-fail" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" 60000 task_1 + [ "$status" -eq 4 ] + [[ "$output" == *"runtime_timeout"* ]] + [ ! -f "$sd/acked" ] + [ "$(cat "$sd/calls")" = "1" ] +} + @test "a payload delivered as an object (not a JSON string) is still classified" { run env ORCA_WAIT_DRYRUN=1 \ ORCA_WAIT_CHECK_JSON='{"result":{"count":1,"deliveryId":"d5","messages":[{"id":"m","type":"worker_done","subject":"s","payload":{"outcome":"failed","taskId":"task_o"}}]}}' \ diff --git a/tests/send-prompt.bats b/tests/send-prompt.bats index 765cd62..747b44e 100644 --- a/tests/send-prompt.bats +++ b/tests/send-prompt.bats @@ -419,10 +419,21 @@ tpl_sections_single_line() { [ "$output" != "2594177116 1010" ] } -@test "template: the Orca prompt set is byte-identical (out of scope for this change)" { +@test "template: the Orca prompt set is byte-identical" { + # Bumped from 1714004932/4937 when rule [4] gained the ask-timeout contract: + # a timeout leaves the question pending, so the worker resumes it instead of + # deciding it. Update in the SAME commit as any intentional edit, as above. run sh -c "sed -n '/^\*\*Orca substrate\.\*\*/,/^## Subagent usage protocol/p' '$TPL' | cksum" [ "$status" -eq 0 ] - [ "$output" = "1714004932 4937" ] + [ "$output" = "3932390147 5667" ] +} + +@test "template: the Orca ask rule forbids deciding a timed-out question" { + # Names the rule the checksum above only pins, so a reword that keeps the + # command but drops the obligation is visible in this test's diff. + grep -qF 'A timeout is not an answer.' "$TPL" + grep -qF 'ask --resume ' "$TPL" + grep -qF 'Do NOT decide it yourself' "$TPL" } @test "template: the REQUIRED block still states its three rules" { From 1839469519292790c477339c9aa0989d49e66e0c Mon Sep 17 00:00:00 2001 From: Younggi Choi <74581798+choiyounggi@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:23:22 +0900 Subject: [PATCH 2/2] fix(orchestrate): put the exit-4 line on stdout and correct the immediate-return list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review of the previous commit found two defects in it. The outage line was the only primary exit-classification message written to stderr; exits 2, 3, 5 and 6 all write theirs to stdout. A coordinator that reads stdout — which is where `ack ` and `completed=/` already go — would have received a bare exit 4 with no error code to act on. Moved to stdout and pinned with a test that discards stderr, since bats merges the two streams and could not otherwise tell them apart. The `--until-all` contract still listed 3/5/6 as the codes that return immediately; exit 4 does too. Corrected in both the script header and SKILL.md, and the exit-4 entry now carries the same already-acked-batches caveat exit 2 has. The "(3/5/6) are deliberately NOT acked" sentence is left alone: it is about Deliveries, and exit 4 never receives one. 387/387 bats green. Co-Authored-By: Claude Opus 5 --- skills/orchestrate/SKILL.md | 2 +- skills/orchestrate/scripts/orca-wait.sh | 8 ++++++-- tests/orca-wait.bats | 12 ++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/skills/orchestrate/SKILL.md b/skills/orchestrate/SKILL.md index caf20b9..0c41a27 100644 --- a/skills/orchestrate/SKILL.md +++ b/skills/orchestrate/SKILL.md @@ -205,7 +205,7 @@ you instead of making you poll. Replace steps 1–3 below with O1–O5: `worker_done` for one of your ids* and the `completed=/` line is scoped to this Wave. Add `--until-all` to keep consuming batches until every listed id is completed, so one Wave costs one coordinator turn instead of one per batch; it - still returns immediately on 3/5/6. Codes 3/5/6 leave the batch unread on + still returns immediately on 3/4/5/6. Codes 3/5/6 leave the batch unread on purpose, so an unhandled event is never silently dropped — which also means delivery is **at-least-once**: a replayed batch must be processed idempotently (key off `taskId`, never off a local counter). `ORCA_WAIT_RECHECK_MS` (default diff --git a/skills/orchestrate/scripts/orca-wait.sh b/skills/orchestrate/scripts/orca-wait.sh index d42c301..130f953 100755 --- a/skills/orchestrate/scripts/orca-wait.sh +++ b/skills/orchestrate/scripts/orca-wait.sh @@ -9,7 +9,8 @@ # usage: orca-wait.sh [--until-all] [task_id,task_id,...] # --until-all keep consuming successful worker_done batches until every task # id listed below is completed, then exit 0 once. Requires the -# task id list. Exit 3/5/6 still return immediately, unacked. +# task id list. Exit 3/5/6 still return immediately, unacked, and +# so does exit 4 (which never received a batch to leave unacked). # total budget for this call (real tasks run 15-60 min); with # --until-all it covers ALL batches and is never reset per batch # [task ids] if given, also report how many of exactly THOSE tasks are @@ -26,6 +27,9 @@ # treat it as a checkpoint: check `orca status --json` first. The workers # themselves may well still be alive; a dead runtime does not stop a # worker session, so do not restart anything on this code alone. +# (--until-all: like exit 2, batches already acked in this call stay +# consumed; this window itself received nothing, so nothing is left +# unacked by it.) # exit 5 an escalation is pending — resolve it, then re-run # exit 6 a worker question is pending — `orchestration reply --id `, re-run # @@ -153,7 +157,7 @@ while :; do if [ "$rc" -ne 0 ] || [ "$ok" = false ] || [ "$lost" = true ]; then ecode=$(printf '%s' "$out" | "$JQ" -r '.error.code // empty' 2>/dev/null) || ecode="" [ "$lost" = true ] && [ -z "$ecode" ] && ecode="connection_lost" - echo "[orca-wait] orca check failed (${ecode:-no response}, status $rc) — the runtime is not answering, not the workers being quiet; run \`orca status --json\` before waiting again" >&2 + echo "[orca-wait] orca check failed (${ecode:-no response}, status $rc) — the runtime is not answering, not the workers being quiet; run \`orca status --json\` before waiting again" exit 4 fi diff --git a/tests/orca-wait.bats b/tests/orca-wait.bats index bbc1445..9942e38 100644 --- a/tests/orca-wait.bats +++ b/tests/orca-wait.bats @@ -173,6 +173,18 @@ teardown() { rm -f "$(REPO_TMP)/orca-stub-${BATS_TEST_NUMBER}"; } [ ! -f "$sd/acked" ] } +# bats merges stdout and stderr into $output, so the stream has to be proven by +# discarding stderr. It matters: every other exit-classification line here is on +# stdout, and a coordinator reading only stdout would otherwise get a bare 4 with +# no error code to act on. +@test "the outage line is on stdout, like every other classification line" { + run sh -c "ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_RC=1 \ + ORCA_WAIT_CHECK_JSON='{\"ok\":false,\"error\":{\"code\":\"runtime_unavailable\"}}' \ + bash '$OW' 60000 2>/dev/null" + [ "$status" -eq 4 ] + [[ "$output" == *"runtime_unavailable"* ]] +} + @test "connectionLost mid-wait exits 4 even though ok:true and count 0 (boundary)" { run env ORCA_WAIT_DRYRUN=1 \ ORCA_WAIT_CHECK_JSON='{"ok":true,"result":{"count":0,"messages":[],"timedOut":false,"connectionLost":true}}' \