diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e4adabb..4cce910 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "simple-workflow", "description": "The Claude Code plugin for an end-to-end AI development workflow — ticket management, multi-agent code review, security audit, and automated PR creation — built on a Harness for long-running AI agents with strict context management and cross-session learning.", - "version": "8.4.2", + "version": "8.5.0", "author": { "name": "aimsise", "url": "https://github.com/aimsise" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b00d430..eb4c62d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Run ShellCheck on hooks - run: shellcheck --severity=warning hooks/*.sh + run: shellcheck --severity=warning hooks/*.sh hooks/lib/*.sh - name: Run ShellCheck on tests run: shellcheck --severity=warning tests/*.sh tests/helpers/*.sh diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d16f563..70924ae 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -15,7 +15,7 @@ Treats the context window as a consumable resource and systematically conserves A **Generator** writes code, independent **Evaluators** verify it, and failures trigger automatic retry with specific feedback — up to 9 rounds by default (configurable per invocation or per ticket via policy). The information firewall is asymmetric: Evaluators never see the Generator's self-assessment and judge solely from `git diff` and test results, while the Generator does receive Evaluator feedback on retry. -Even though both sides run the same model, **weights × context = output** — by excluding the Generator's trial-and-error history from the Evaluator's context, sunk-cost bias is structurally eliminated rather than merely discouraged by prompt. FAIL-CRITICAL violations halt execution immediately, and after ticket completion evaluation logs feed into the Knowledge Base, closing a cross-session feedback loop. +The two sides may run different models by policy — the Generator is always opus, while Evaluators are sonnet by default and escalate to opus for critical or exhaustive work — so independence is enforced by **context separation (the information firewall)**, not by weight uniformity: excluding the Generator's trial-and-error history from the Evaluator's context eliminates sunk-cost bias structurally rather than merely discouraging it by prompt. FAIL-CRITICAL violations halt execution immediately, and after ticket completion evaluation logs feed into the Knowledge Base, closing a cross-session feedback loop. ## Knowledge Base (Cross-Session Learning) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3259d9..85beb5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.5.0] — 2026-06-20 + +**TL;DR.** Adds **Advertised-Accept-Set Conformance (AASC)** — a product-/language-/domain-agnostic meet-or-beat upgrade to verification. When a boundary advertises **strict / canonical / lossless / limit** (or shares an input class with a sibling), the `ac-evaluator` now **EXECUTES** a generative grammar-complement sweep in scratch and diffs the unit's accept-set against an **independent hand-coded oracle** — catching parse-accepted overflow, input-alphabet leaks (incl. astral Unicode digits), non-canonical accepts, and structural-key / prototype-pollution injection **by construction for all inputs**, instead of by recalling a product-specific keyword cue. A new recognition-independent PostToolUse hook (`hooks/accept-set-verify.sh`) deterministically gates the persisted sweep and **enforces by default**. This release also codifies the product/language/domain-**agnosticism** rules (the (A)/(B) substrate line + the meet-or-beat principle + the `CT-DECONTAM-1` product-instance recidivism guard) and lands the P0 safety / state-machinery hardening batch. Validated across dogfood43→53: a deliberately leak-inviting strict subject produced correct strict artifacts on every historically-failing trap, and a live `decision:block` de-risk confirmed the enforce path. Additive + kill-switched: `constraints.accept_set_conformance: off` or `SW_ACCEPT_SET_CONFORMANCE_MODE=off` reverts byte-for-byte to the pre-v8.5.0 read-only strictness reasoning. + +### Added + +- **Advertised-Accept-Set Conformance — the executed accept-set sweep.** For an AC whose boundary advertises strict / canonical / lossless / limit OR belongs to a `shared_input_boundary` sibling family, the `ac-evaluator` derives a per-boundary **Grammar Card** (`A` alphabet / `U` unicode-transform / `W` canonical-writer / `K` keyed-structure) and EXECUTES four machine-generated metamorphic relations black-box in `.simple-workflow/scratch/` against an independent hand-coded spec oracle: **MR-FINITE** (parse-accepted overflow), **MR-ALPHABET** (the Unicode decimal-digit-property complement across the BMP **and** astral planes — generator names no script/codepoint), **MR-CANONICAL** (non-canonical accept), **MR-KEYFAITH** (a reserved/accessor/private-slot key derived by reflection over the type/prototype — generator names no key literal). A divergence is force-FAILed only under a two-tier oracle-authoritative gate (otherwise advisory `[MEDIUM]`, fail-open where no runnable artifact exists). One unconditional `## Accept-set sweep` line per inspected boundary is persisted to `eval-round-{n}.md` (`boundary= triggered= ran= astral= corpus-size= divergences= authoritative= caveat=`). A worked copyable shape lives in `skills/impl/references/accept-set-conformance-harness.md`; committed proof prototypes in `design-oracles/aasc-accept-set/` and `design-oracles/aasc-keyfaith/` (verified `node` + `python3`). Guarded by **CT-AASC-1..14**. +- **Per-AC deterministic AASC trigger + `constraints.accept_set_conformance` kill switch.** `/impl` Step 15 computes, per AC, whether the boundary advertises strict/canonical/lossless/limit (lexical) OR is a shared-input sibling (a keyed-structure-from-untrusted-input always triggers the K axis), inlines `Accept-set conformance: {auto|off} triggered-on={ids}` into the evaluator spawn, and emits `[ACCEPT-SET-TRIGGER]` — so the evaluator never re-decides whether to run the sweep (removing the run-to-run recognition variance). The per-brief `constraints.accept_set_conformance` field is documented symmetrically across all three spawner surfaces (`policy-template.md`, `autopilot-policy-reference.md`, `ac-evaluator-orchestration.md`); **CT-AASC-14**. +- **AASC determinism hook (`hooks/accept-set-verify.sh`).** A recognition-independent `PostToolUse(Write|Edit)` gate that reads the persisted `## Accept-set sweep` line and applies the lens's own self-incrimination rule with zero model recall: a triggered boundary not run (P1), an A/U-axis sweep that skipped the astral complement (P2), or an authoritative divergence not driven to FAIL (P4) emits a `decision:block`; a thin A/U corpus (P3) is advisory-only. Fail-OPEN (exit code always 0; jq-missing / non-eval path / skeleton / `n/a` all pass). Env knobs `SW_ACCEPT_SET_CONFORMANCE_MODE` + `SW_AASC_CORPUS_FLOOR`. Verified by `tests/test-accept-set-verify.sh` (**32/32**) and registered via **CT-AASC-13**. +- **Product/language/domain agnosticism codification.** `CLAUDE.md` gains the **(A) user-product / (B) harness-own substrate line**, the *agnosticism must never lower quality* (meet-or-beat) rule, and the HARD-LINE-vs-JUDGMENT enforcement boundary. **CT-DECONTAM-1** is a product-instance recidivism guard over the 14-file normative set (denylist = product instances only; abstract failure-class names + bare domain vocabulary are deliberately kept), and the normative content was de-contaminated of product-specific cues. +- **Cross-ticket `shared_input_boundary` sibling-guard (P-A).** The planner / ticket-evaluator forward a shared-input-class signal so a delegating sibling's boundary is recognized (it triggers the accept-set sweep even with no lexical strict/canonical word present). +- **Safety / state-machinery hardening (P0 batch).** New `hooks/pre-bash-contract-guard.sh` (Bash-mediated state-mutation guard, `SW_BASH_STATE_GUARD_MODE`); jq-missing (`SW_SAFETY_JQ_MISSING_MODE`) and HOOK_OWNED_FIELDS (`SW_STATE_FIELD_GUARD_MODE`) guards across `pre-bash`/`pre-write`/`pre-edit-safety`; `hooks/lib/parse-state-file.sh` element-scoped state parsers; `autopilot-continue` loop-guard de-pollution + `post-ship-state-auto-compact` Gate 5.5 integrity self-heal (`SW_POST_SHIP_INTEGRITY`). New suites `tests/test-pre-bash-contract-guard.sh`, `tests/test-state-parsers.sh`, `tests/test-autopilot-continue.sh` + safety-guard additions. + +### Changed + +- **AASC determinism hook promoted to enforce (`on`) by default.** After dogfood51/52 (14 real conformant eval reports → 0 false-trips) + a live `decision:block` de-risk (dogfood53: the block surfaced cleanly, the model handled it gracefully without thrash and was not pressured into fabricating a conformant line), the `SW_ACCEPT_SET_CONFORMANCE_MODE` default flipped `metric-only` → `on`. Set `metric-only` to observe-only or `off` to disable. Non-breaking: this is the initial default of a new feature with a kill switch, not a change to any prior released behaviour. + +### Fixed + +- **AASC hook robustness (dogfood51/52 confirmation + pre-release audit).** The `## Accept-set sweep` header is now matched at any hash depth (`#{1,6}`) and case-insensitively (was `## `-only / case-sensitive, which let a mis-leveled / mis-cased header skip a whole report); fields parse whitespace-bounded (tab-separated lines no longer dodge); the line selector is order-independent; an off-grammar boundary label emits a stderr WARN (not a block). The **P3 corpus floor was demoted from blocking to advisory** — corpus-size is a weak depth proxy and flooring it false-tripped legitimately-thin-but-conformant sweeps; the astral check (P2) is the real A/U depth gate. +- **EC-taxonomy uniformity.** `evidence-channels.md` no longer carries the stale "MR-KEYFAITH is ASSUMED / advisory-only" posture (folded into the same two-tier oracle-authoritative FAIL gate as the other relations, matching the `ac-evaluator` twins + harness doc); **CT-AASC-10** now also scans `evidence-channels.md` so the sibling-artifact uniformity rule is mechanized. + +### Verification + +- `bash tests/test-skill-contracts.sh` **852/852** (+ **CT-AASC-1..14**, **CT-DECONTAM-1**, **CT-EV-PANEL-ROBUST**, twin byte-identity **CT-EV-MODEL-1**); `bash tests/test-accept-set-verify.sh` **32/32**; `bash tests/test-path-consistency.sh` **144/144**; `shellcheck --severity=warning` clean on the new hook + tests; `ac-evaluator` / `ac-evaluator-hi` bodies byte-identical sans `name:` / `model:`. `plugin.json` `8.5.0` == newest CHANGELOG `[8.5.0]` (CT-MODE-14), real ISO date (CT-MODE-13). +- **Dogfood evidence.** dogfood52 (a leak-inviting strict-numeric + keyed + canonical/delegating subject built on the exact historically-failing traps) → the harness produced correct strict artifacts on every trap (impl explicitly avoided the `\d`-without-`re.ASCII` Unicode-digit trap), the AASC chain fired on all 3 tickets with genuinely-executed astral-inclusive sweeps and zero stand-downs, and an independent re-probe found 0 leaks; dogfood53 → a live `decision:block` on a fault-injected non-conformant report, handled gracefully. +- **Additive + kill-switched.** `constraints.accept_set_conformance: off` (or `SW_ACCEPT_SET_CONFORMANCE_MODE=off`) reverts byte-for-byte to the pre-v8.5.0 read-only strictness reasoning; the determinism hook is fail-OPEN (exit code always 0) and structurally cannot remove a catch the prior harness had. The safety/state hardening guards ship at their documented defaults with `SW_*_MODE` kill switches. + ## [8.4.2] — 2026-06-11 **TL;DR.** Fixes a manual-mode dead-end on the `/autopilot` path. A brief created with `chain: off` (manual mode) intentionally never receives a per-ticket `autopilot-policy.yaml` (Step W-8 skips propagation), yet `/autopilot` Phase 1 step 3 used to `[WARN] ... only brief-level policy is in effect` and continue — after which the three per-ticket Policy guards (which have no brief-level fallback) aborted every ticket. `/autopilot` on a manual brief now hard-stops once at the entry with an actionable re-propagation directive, and `/brief`'s manual-flow guidance names the prerequisite before advertising the autopilot switch. Surfaced by the follow-up #2 live `/autopilot` validation; the original "`/create-ticket` policy-copy is broken" framing was a non-bug (W-8's manual skip is spec-correct and `[POLICY-PROPAGATION] skipped: brief mode=manual`-pinned) — the real defect was the downstream WARN-and-continue that stranded the run. diff --git a/CLAUDE.md b/CLAUDE.md index eba5bbe..c50468c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,6 +33,11 @@ This rule was distilled from a v6.7.0 dogfood incident in which a verify hook ne - `SW_AUTOPILOT_ASK_GUARD` — controls `hooks/pre-askuserquestion-guard.sh`. Values: `on` (default; matrix active), `metric-only` (compute the matrix and log `[ASK-GUARD] metric-only: would deny ...` to stderr without denying), `off` (disable the guard; unknown values collapse here so a typo fails open). The hook implements the 3-tier `risk_tolerance` allow-list documented in `skills/autopilot/SKILL.md ## Non-interactive orchestrator contract (3-tier, risk_tolerance-aware)`; when `autopilot-policy.yaml` is absent or carries an unknown `risk_tolerance:` value, `hooks/lib/parse-state-file.sh::get_risk_tolerance` returns `conservative` (deliberate fail-open so the six known gate-id headers remain answerable). - `SW_AUTOPILOT_DONE_GATE_TTL_SEC` — default `86400` (24 hours). TTL window for `hooks/scout-checkpoint-guard.sh` Step 2a (the autopilot-completion gate added in v8.0.1). The gate silent-exits the hook when (1) no `autopilot-state.yaml` exists under `briefs/active/` or `product_backlog/` (no active autopilot), (2) `briefs/done//autopilot-state.yaml` exists with mtime within this TTL, AND (3) every `tickets[].status` equals `completed`. Required because the existing Step 3 phase-state.yaml lookup uses `find_phase_state_file` which scans `active/` only — after `/ship` moves the brief to `briefs/done/`, the moved `phase-state.yaml` becomes invisible and the 3-AND (a)(b)(c) would otherwise false-block on stale transcript artifacts. Set `0` to disable the TTL bound (the gate fires on any all-completed `briefs/done/` state regardless of age — useful for replay scenarios where a session is resumed long after autopilot finished). Shorten (e.g. `600`) during dogfood / regression debugging when you want stale done state to fall through immediately. Non-numeric values fall back to `86400` with a one-line `[SCOUT-AUTOPILOT-DONE-GATE]` stderr warning. Honoured only when `SW_SCOUT_CHECKPOINT_MODE` is not `off` — the kill switch in Step 2 short-circuits the entire hook including Step 2a. - `SW_AUTOPILOT_POLICY_STOP_HONOR` — default `on`. Controls whether the three autopilot Stop hooks (`hooks/autopilot-continue.sh`, `hooks/impl-checkpoint-guard.sh`, `hooks/scout-checkpoint-guard.sh`) HONOUR a model-declared hard-stop instead of forcing the loop to continue. When the orchestrator legitimately hard-stops it emits `[AUTOPILOT-POLICY] gate= action=stop reason=<...>` in its last assistant turn (mandated by `skills/autopilot/SKILL.md`); the shared detector `hooks/lib/detect-policy-gate-stop.sh::last_turn_declares_policy_gate_stop` scans only that LAST assistant turn's `text` content blocks for the marker. Values: `on` (default — honour: `autopilot-continue.sh` allows `end_turn` and appends a `boundary: session_end` / `stop_reason: policy_gate_stop` `runtime_metrics` entry, the two checkpoint guards stand down without blocking and clear their own counter, none of the three re-injects "Do NOT stop"), `metric-only` (detect and log a `[POLICY-GATE-STOP]`-prefixed stderr line stating it WOULD honour, but still emit `decision: block` — forensic mode), `off` (ignore the declaration entirely; block exactly as before). Unknown values fail **closed** to `off` semantics so a typo never silently widens the honour behaviour. This knob is purely additive: with it `off` OR no marker present, the existing loop guards (autopilot-continue's `FILE_COUNT>=5 && NOTOOL_COUNT>=5`; the checkpoint guards' 3× counters) remain the sole backstop and every hook behaves exactly as before. Detection is independent of `AUTOPILOT_LEGACY_LOOPGUARD`. Fixes a dogfood thrash where a hard-stop was re-injected 11× because the hooks ignored the model's declared `policy_gate_stop`. +- `SW_SAFETY_JQ_MISSING_MODE` — default `metric-only`. Controls the jq-missing preflight at the top of the three safety hooks (`hooks/pre-bash-safety.sh`, `hooks/pre-write-safety.sh`, `hooks/pre-edit-safety.sh`). `jq` is a documented hard dependency; when it is absent these guards cannot parse the tool payload and previously died with a silent `exit 127` (which Claude Code treats as a non-blocking error — a fail-OPEN). Values: `on` (fail **closed** — `exit 2` with an explicit `[SAFETY-JQ-MISSING] : jq not found …` message, blocking the tool call), `metric-only` (default — log `[SAFETY-JQ-MISSING] metric-only: … would fail closed …` to stderr and ALLOW the call with `exit 0`, so the shipped fail-open behaviour is unchanged but now observable), `off` (silently allow, `exit 0`, no message). Unknown values collapse to `metric-only` (the observe-only default — a typo never silently flips the guard to fail-closed). **Promotion**: ship at `metric-only`; after one dogfood confirms no false trips on a jq-present host, set `=on` to enforce fail-closed. The `[SAFETY-JQ-MISSING]` exit-2 path is verified under a PATH-restricted (no-jq) test in `tests/test-pre-bash-safety.sh` / `tests/test-pre-write-safety.sh` / `tests/test-pre-edit-safety.sh` with the knob set to `on`. +- `SW_BASH_STATE_GUARD_MODE` — default `metric-only`. Controls detection 3 of `hooks/pre-bash-contract-guard.sh` (the autopilot-context PreToolUse:Bash guard): a Bash-mediated state-file status mutation — `yq -i` / `sed -i` / shell-redirect on `autopilot-state.yaml` / `phase-state.yaml` that drives a `status:` (or `steps.`) to `skipped` / `completed` / `failed` / `in_progress`. The Write/Edit skip-transition guard (`hooks/pre-state-transition.sh`) is wired only to Write/Edit, so this closes the Bash bypass (ST-04). Values: `on` (emit `decision:block` with `unauthorized_state_mutate_bash` — deny the call), `metric-only` (default — log `[PRE-BASH-CONTRACT-GUARD] metric-only: would deny ...` to stderr and ALLOW), `off` (skip detection 3 entirely). Unknown values collapse to `metric-only`. Detections 1 (forbidden manual_bash_fallbacks rationale) and 2 (direct `git commit`) are NOT gated by this knob — the hook's existing NAC posture is preserved. **Promotion**: ship at `metric-only`; after one dogfood confirms no false trips, set `=on` to enforce. Read-only commands (`grep`/`cat`/`yq` without `-i`) and `runtime_metrics` appends (no status transition) are not matched. +- `SW_STATE_FIELD_GUARD_MODE` — default `metric-only`. Gates the **HOOK_OWNED_FIELDS enforcement** in `hooks/pre-write-safety.sh` / `hooks/pre-edit-safety.sh`. The registry now ships with `.runtime_metrics` (an append-only telemetry list written exclusively by the six runtime-metrics writer hooks — Foundation 3 / ST-03), so a model Write/Edit that would clobber hook-appended `runtime_metrics` entries (a lost-update) is detectable. Values: `on` (emit `decision:block` whose reason NAMES the violated field — e.g. `.runtime_metrics` — and references `docs/state-schema.md`), `metric-only` (default — log `[STATE-FIELD-GUARD] metric-only: would block ...` to stderr and ALLOW, so populating the registry does NOT change the shipped allow-by-default behaviour), `off` (allow silently). Unknown values collapse to `metric-only`. **Promotion**: ship at `metric-only`; after one dogfood confirms no false trips, set `=on` to enforce. Detection covers the inline `runtime_metrics: []` → `[…]` (and blank-out) form; a multi-line list mutation expressed without an inline value change is not detected (acceptable under the metric-only default). +- `SW_ACCEPT_SET_CONFORMANCE_MODE` — default `on` (v8.5.0). Controls `hooks/accept-set-verify.sh`, the PostToolUse(Write|Edit) gate that deterministically verifies the persisted `## Accept-set sweep` section of an `eval-round-{n}.md` report (the Advertised-Accept-Set Conformance observability line written by `agents/ac-evaluator.md`'s Persistence-First terminal rewrite). The hook reads the EMITTED line and applies the lens's OWN self-incrimination rule with zero model recall — BLOCKING on a triggered boundary not run (P1), an alphabet/unicode (A/U-axis) sweep that skipped the astral complement (P2), or an authoritative divergence not driven to FAIL (P4); plus a non-blocking ADVISORY note when an A/U-axis corpus falls below `SW_AASC_CORPUS_FLOOR` (P3 — corpus-size is a weak depth proxy, so a thin corpus is surfaced, NEVER blocked; the dogfood51 confirmation run showed flooring it false-trips a legitimately-thin-but-conformant sweep, so P3 was demoted to advisory). The keyed (K) / canonical-writer (W) axes are exempt from the astral/corpus checks (a reflection-derived key corpus is legitimately small), `caveat=no-runnable-artifact` exempts P1/P2 and the P3 advisory (a compiled-language fail-open degradation), and the `## Accept-set sweep` header is matched case-insensitively (a mis-cased header cannot let a whole report skip the gate). Values: `on` (DEFAULT, v8.5.0 — enforce: emit a PostToolUse `decision:block` whose reason names the violated predicate, surfacing it to the evaluator to re-run the sweep), `metric-only` (observe — log `[ACCEPT-SET-VERIFY] metric-only: would block ...` to stderr and ALLOW), `off` (explicit opt-out, silent). Unknown values collapse to `metric-only` (since the default is `on`, the var is set only to DOWNGRADE, so a typo on a downgrade value falls back to the safe observe mode — never a surprise enforce, never a silent disable). The exit code is ALWAYS 0 (fail-OPEN — the hook can never break a Write/Edit; the only non-allow influence is the `on`-mode block JSON), and a missing `jq` is a silent `exit 0`. **Promoted to `on` by default in v8.5.0** after dogfood51/52 (14 real conformant reports → 0 false-trips) + a live `decision:block` de-risk (dogfood53: the block surfaces cleanly and the evaluator handles it gracefully without thrash, and is NOT pressured into fabricating a conformant line); set `metric-only` to revert to observe-only, `off` to disable. Verified by `tests/test-accept-set-verify.sh`. This is the runtime, recognition-independent half of the per-brief `constraints.accept_set_conformance` switch (which gates whether the EXECUTED sweep runs at all). +- `SW_AASC_CORPUS_FLOOR` — default `256`. The ADVISORY threshold for the P3 thin-corpus NOTE in `hooks/accept-set-verify.sh` (a thinner A/U corpus is surfaced to stderr as `[ACCEPT-SET-VERIFY] advisory: ...`, NEVER blocked), applied ONLY to a `triggered=y ran=y` alphabet (A) or unicode-transform (U) boundary line — the axes whose mandated complement (the Unicode decimal-digit property across the BMP and astral planes) is naturally large. A descriptive corpus-size (e.g. `5-canonical-forms`) is parsed by its leading integer so an annotation cannot dodge the note. Lower it (e.g. `=8`) to silence the note on a legitimately small advertised accept-set; raise it to surface more sweeps. Honoured only when `SW_ACCEPT_SET_CONFORMANCE_MODE` is not `off`. ## Language @@ -40,6 +45,59 @@ This rule was distilled from a v6.7.0 dogfood incident in which a verify hook ne Non-English content is allowed only in gitignored paths: `.simple-workflow/`, `.docs/`, `CLAUDE.local.md`, `.claude/settings.local.json`, `.claude/worktrees/`, `.env*`. +## Product/Language/Domain Agnosticism + +**IMPORTANT**: the plugin's normative content — everything under `skills/`, +`agents/`, and `hooks/` that instructs the model how to build, verify, or ship — +MUST serve every user's needs without baking in dependence on a specific +product, programming language, or problem domain. Triggers, rubrics, gates, and +examples MUST key off **properties** (a computed numeric/algorithmic value; a +shared input/validation boundary; an irreversible side-effect; a behaviour the +spec advertises as strict / canonical / lossless / or bounded by a limit), never +off membership in a named-product / named-language / named-domain list. + +### The (A) / (B) substrate line + +Two substrates, two different rules: + +- **(A) The user-product substrate** — the source, tests, and artifacts a user + builds *with* the plugin. This MUST be treated agnostically. The harness may + not assume the user's language, framework, test runner, or domain. A named + language / framework / domain may appear ONLY as clearly-labelled, + non-exhaustive **illustration** ("e.g. …", "illustrative only") — never as the + operative trigger, an exhaustive checklist, or a recognition cue the mechanism + depends on. +- **(B) The harness-own substrate** — the plugin's *own* operating dependencies: + `git`, `gh`, `jq`, `yq`, the Claude Code runtime (hooks, skills, agents, the + `Agent` / `Workflow` tools, the state-file machinery), and the PR / release + delivery path. These are legitimately FIXED and are NOT subject to the (A) + rule. Depending on `git` for repo inspection or `gh` for PR creation is + harness plumbing, not a domain dependency. + +### Agnosticism must never lower product quality + +Agnosticism constrains *how* the harness is written; it is not a licence to +verify less. The harness (subagents / skills / hooks) MUST **meet or beat** its +prior defect catch-rate. Where an agnostic formulation cannot guarantee the same +quality a product/domain-specific one did, the answer is NOT to retreat to the +specific cue: thoroughly explore and **generate** an innovative agnostic +mechanism, **verify** it catches the baseline defect classes by construction, +gain confidence, and only THEN implement and update the plan. A property- / +grammar- / oracle-driven mechanism that catches a defect class for ALL inputs is +strictly stronger than a keyword cue that catches it only when a known token is +recalled. + +### Enforcement boundary (HARD-LINE vs JUDGMENT) + +Normative-prose agnosticism is primarily a **review standard**, not a fully +grep-enforced invariant. Only specific, stable **product-instance token** +denylists are mechanised as contract tests (e.g. `CT-DECONTAM-1`, whose +denylist is product instances ONLY — abstract failure-class names and bare +domain vocabulary are deliberately excluded). Judgment criteria ("is this +example illustrative or load-bearing?") MUST NOT be faked as contract tests. +When in doubt, prefer a property-based formulation and leave the final +agnosticism call to review. + ## Releases - SemVer with the `v` prefix (`v6.0.0`, never `6.0.0`). @@ -108,3 +166,5 @@ For any intentional omission, record a one-line rationale in the commit message - Triple-backtick fenced code blocks are exempt from the absolute-home-path scan, so verbatim CI logs (e.g. `/Users/runner/work/...`) may be quoted inside fenced blocks. Do not abuse this exemption for project-authored paths. - The `.gitignore` file is allowlisted because it legitimately stores absolute paths. - The detection regex is case-sensitive and POSIX-only: lowercase `/users/` and Windows-style `C:\Users\...` paths are out of scope. +- **Known limitation — unclosed fences**: the scan toggles fence state on each fence line (a line beginning with three backticks), so an **unclosed** (odd-count) fence extends to end-of-file per CommonMark and exempts every following line from the absolute-home-path scan. An accidental unbalanced fence in generated Markdown can therefore let a home path slip past the guard, which does not validate fence pairing — keep fences balanced. +- **Known limitation — Bash write sinks**: the PII / sensitive-file guards run only on the `Write` and `Edit` tools. The same content written through `Bash` (`echo > file`, `tee`, `python3 -c`, a heredoc) is **not** inspected — `pre-bash-safety.sh` checks `git add` staging plus the destructive / network / identity / privilege denylist, not write sinks. A PII string or sensitive filename created via a shell redirect bypasses these guards by design (best-effort defense-in-depth); prefer `Write` / `Edit` for files that may carry PII. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b5a3623..ccdd8d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,6 +65,8 @@ simple-workflow/ 3. Register it in `hooks/hooks.json` 4. Add tests in `tests/test-your-hook.sh` +Shared logic that several hooks source belongs in `hooks/lib/.sh` instead. Library helpers are *sourced*, so they do **not** carry `set -euo pipefail` (that would leak into the calling hook's shell); they are standalone files that export functions only, with no top-level side effects. See **Code Style** below. + ### Measuring SKILL.md Size For compression work, measure token count with `tests/helpers/count-tokens.sh` rather than bytes: @@ -91,14 +93,14 @@ Tests follow the pattern in `tests/test-helper.sh`, using `assert_allowed`, `ass ## Code Style -- Shell scripts must pass [ShellCheck](https://www.shellcheck.net/) (`--severity=warning`) -- Use `set -euo pipefail` at the top of all shell scripts +- All shell scripts must pass [ShellCheck](https://www.shellcheck.net/) at `--severity=warning`. CI lints `hooks/*.sh`, `hooks/lib/*.sh`, `tests/*.sh`, and `tests/helpers/*.sh`; run the same scope locally before opening a PR (see Pull Request Process below). Silence an intentional warning with a targeted `# shellcheck disable=SCxxxx` directive placed in front of the affected command — for a `case` arm the directive goes in front of the whole `case` statement, not the individual branch — never by loosening the severity. +- **Hook entry scripts** (`hooks/*.sh`) start with `#!/usr/bin/env bash` and `set -euo pipefail`. **Library helpers** (`hooks/lib/*.sh`) are *sourced* by those entry scripts and intentionally do **not** set shell options at the top: a `set -euo pipefail` in a sourced file is a top-level side effect that would mutate the caller's shell. Library helpers are standalone files that export functions only, with no top-level side effects (each helper that manipulates YAML applies a `yq` → `python3 + PyYAML` → pure-shell fallback); the calling hook owns the shell flags. - Use conventional commit prefixes: `feat:`, `fix:`, `docs:`, `test:`, `chore:` ## Pull Request Process 1. Ensure all tests pass: `bash tests/run-all.sh` -2. Ensure ShellCheck passes: `shellcheck --severity=warning hooks/*.sh tests/*.sh` +2. Ensure ShellCheck passes: `shellcheck --severity=warning hooks/*.sh hooks/lib/*.sh tests/*.sh tests/helpers/*.sh` (the same scope CI lints) 3. Create a PR to `main` with a clear description 4. Fill out the PR template diff --git a/README.md b/README.md index 81f634e..22c482c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,30 @@ The [Claude Code](https://docs.anthropic.com/en/docs/claude-code) plugin for **end-to-end AI development workflows**. From idea to pull request: structured interview, codebase investigation, multi-agent implementation, security audit, code review, and PR creation, all automated. -Built on a **Harness for long-running AI agents** with strict context management, information firewalls, and a cross-session knowledge base that improves accuracy with every completed ticket. +Built on a **Harness for long-running AI agents** that brings *loop engineering* to the development lifecycle — a rigorous, closed *inner loop* (act → verify → correct → continue) bounded by contract stopping conditions — with strict context management, information firewalls, and a cross-session knowledge base that improves accuracy with every completed ticket. + +

+ + + How /brief chain=on runs: /brief then /create-ticket then /autopilot, then a per-ticket loop of /scout, /impl (with an inner verify loop) and /ship, ending in pull requests + +
+ The default /brief chain=on flow at a glance: skills (blue) drive subagents (green); the per-ticket loop and the in-/impl verify loop are red; lifecycle hooks drive and guard every step. The full flow — agents · hooks · harness — is in the collapsible below. +

+ +
+Full flow — every skill, agent, hook & harness mechanism + +

+ + + Detailed /brief chain=on execution flow: each phase's work, subagents and artifacts; the handoff and harness applied between phases; the lifecycle hooks; the per-ticket and verify loops; and a summary of the harness mechanisms + +

+ +Regenerate: python3 docs/gen-brief-chain-overview.py · python3 docs/gen-brief-chain-flow.py + +
## Prerequisites @@ -51,12 +74,12 @@ claude plugin install simple-workflow@aimsise-simple-workflow --scope project Inside an active Claude Code session, type `/brief ` and the plugin handles the rest end-to-end: codebase investigation, requirements interview, ticket creation, implementation, multi-agent review, and pull request. -Full argument signature: `/brief [mode=auto|manual]` (default `mode=auto`). +Full argument signature: `/brief [chain=on|off]` (default `chain=on`). The `chain=on|off` form is canonical; `mode=auto|manual` is a deprecated legacy alias (`chain=on` ≡ `mode=auto`, `chain=off` ≡ `mode=manual`). | Mode | Command | Result | |------|---------|--------| -| Full automation (default) | `/brief ` | Idea → PR with zero intervention; large scopes are auto-split into multiple tickets and executed in dependency order | -| Brief-assisted manual | `/brief mode=manual` | Structured brief and decision policy are produced; you drive each subsequent step | +| Full automation (default, `chain=on`) | `/brief ` | Idea → PR with zero intervention; large scopes are auto-split into multiple tickets and executed in dependency order | +| Brief-assisted manual (`chain=off`) | `/brief chain=off` | Structured brief and decision policy are produced; you drive each subsequent step (legacy alias: `mode=manual`) | | Resume an interrupted run | `/autopilot ` | Pick up where a previous automated run left off using state files under `.simple-workflow/backlog/` | ### Execution chains @@ -65,24 +88,27 @@ Three typical execution patterns: ```text # 1. Brief-assisted manual (type each command; repeat scout→impl→ship per ticket): -/brief mode=manual +/brief chain=off /create-ticket /scout /impl /ship # → PR -# 2. Brief manually, then hand off to autopilot for the per-ticket loop: -/brief mode=manual -/create-ticket +# 2. Brief manually, then switch to autopilot for the per-ticket loop: +/brief chain=off +# ...inspect the brief, then opt into autopilot: set `chain: on` in the brief file and +# re-run /create-ticket so each ticket dir receives autopilot-policy.yaml (re-propagation): +/create-ticket brief=.simple-workflow/backlog/briefs/active//brief.md /autopilot -# autopilot then loops: /scout → /impl → /ship per ticket → PR +# autopilot then loops: /scout → /impl → /ship per ticket → PR ← loop engineering fires (the closed inner loop self-drives) +# (running /autopilot directly on a chain=off brief stops with a re-propagation directive) # 3. Full automation (one command): -/brief mode=auto -# brief chains: /create-ticket → /autopilot → (per ticket: /scout → /impl → /ship) → PR +/brief +# brief chains: /create-ticket → /autopilot → (per ticket: /scout → /impl → /ship) → PR ← loop engineering fires once /autopilot takes over ``` -> **Caveat — full automation works best on focused, well-scoped ideas.** On overly broad or ambiguous input, the model can break output contracts, fabricate intermediate state, and continue past failures without surfacing them. Full automation has fewer human-in-the-loop checkpoints than the brief-assisted manual flow, so this kind of misbehaviour is easier to miss. For large or exploratory work, prefer `mode=manual` (you inspect artifacts at each step) or split the work into smaller, focused briefs. +> **Caveat — full automation works best on focused, well-scoped ideas.** On overly broad or ambiguous input, the model can break output contracts, fabricate intermediate state, and continue past failures without surfacing them. Full automation has fewer human-in-the-loop checkpoints than the brief-assisted manual flow, so this kind of misbehaviour is easier to miss. For large or exploratory work, prefer `chain=off` (you inspect artifacts at each step) or split the work into smaller, focused briefs. For phase-by-phase workflows on an existing backlog (skipping the brief), run `/help` inside Claude Code to discover the individual slash commands, or browse `skills/` in this repository. @@ -90,9 +116,9 @@ Tickets carry a `### Capabilities` section that records which user Skills and MC ## Why simple-workflow? -simple-workflow stands on three pillars: +simple-workflow stands on three pillars — together they form the closed *inner loop* of *loop engineering* and the engineering around it that makes the loop trustworthy enough to run unattended: -- **Harness Engineering**: structural constraints — an asymmetric information firewall between code authors and code judges (the Generator-Evaluator pattern), bounded sub-agent returns, ticket-confined artifacts, and safe-clear `[SW-CHECKPOINT]` markers — enforce quality by architecture rather than by prompt instructions +- **Harness Engineering**: an asymmetric information firewall between code authors and code judges (the Generator-Evaluator pattern), ticket-confined artifacts, and safe-clear `[SW-CHECKPOINT]` markers are enforced *structurally* — by lifecycle hooks, fresh sub-agent contexts, and on-disk artifacts that hold regardless of model behavior. The bounded sub-agent return budget (< 500 tokens) is a *prompt-level contract* pinned by contract tests, not a runtime-truncation guarantee - **Context Conservation**: the context window is treated as a consumable resource — sub-agents return < 500-token summaries, artifacts live on disk, and state survives compaction - **Cross-session learning**: evaluation logs are distilled into reusable patterns that future implementations inject into their prompts, so the system gets better at your project the more tickets it completes @@ -105,7 +131,7 @@ These pillars exist because Claude Code is powerful, but its context window is f | **Contamination** | Biasing information leaks into contexts where it distorts judgment | Information firewall + ticket directory confinement (see [Harness Engineering](ARCHITECTURE.md#harness-engineering)) | | **Bloat** | Unbounded intermediate output crowds out critical instructions | Artifacts written to files, structured summaries returned to orchestrator | -simple-workflow addresses each threat with architectural constraints that hold regardless of model behavior — not prompt-level instructions that the model might rationalize away. For a deeper walkthrough of each pillar — Context Conservation Protocol, Harness Engineering, Knowledge Base, and Ticket Management state machine — see [ARCHITECTURE.md](ARCHITECTURE.md). +simple-workflow addresses the **structural** threats — Loss, Contamination, Bloat — with architectural constraints that hold regardless of model behavior: automatic snapshots, the information firewall + ticket confinement, and artifacts-to-disk. **Exhaustion** is mitigated by a mix of structural measures (fresh sub-agent contexts, phase-aware context release) and a prompt-level return-size contract (< 500 tokens) pinned by contract tests rather than enforced by runtime truncation. For a deeper walkthrough of each pillar — Context Conservation Protocol, Harness Engineering, Knowledge Base, and Ticket Management state machine — see [ARCHITECTURE.md](ARCHITECTURE.md). ## Setup & Configuration @@ -166,9 +192,12 @@ When injection cannot fire, the hook surfaces a one-line diagnostic via `inject_ - Designed for use with Claude Code CLI. IDE extensions (VS Code, JetBrains) may have limited support for hooks and plugin features. - Pull-request creation requires GitHub CLI (`gh`) with authentication. Other Git hosting services are not supported. - Ticket management uses the local filesystem (`.simple-workflow/backlog/`). There is no sync with external issue trackers (Jira, Linear, etc.). -- Sub-agents consume API tokens independently. Large tickets (L/XL) using Opus may result in higher API costs. +- Sub-agents consume API tokens independently. The Generator (implementer) always runs on Opus, and the evaluator escalates to Opus for critical/exhaustive work, so larger or higher-risk tickets may incur higher API costs. - Built-in test/lint detection covers JS, Python, Rust, Go, JVM (Gradle/Maven/sbt), .NET, Ruby, Elixir, Swift, Flutter/Dart, PHP, and Make. For other ecosystems, wrap your test/lint commands in a Makefile (`make test` / `make lint`) or the evaluator falls back to static code analysis only. - Some recovery paths require interactive mode; running in `claude -p` or CI may stop with an explanatory message rather than complete the recovery. +- **Operating system support**: macOS and Linux are verified (the hook layer is `bash` + `jq`, with optional `yq` / `python3`). Windows is **not** verified — the `bash`+`jq` hook layer requires a POSIX environment (Git Bash, WSL, or Cygwin); native Windows is unsupported. +- **Global command blocks (installation footprint)**: a `v8.0.0` defense-in-depth `Bash` pre-hook unconditionally blocks four command classes in **every** project and session where the plugin is active — network egress (`curl`, `wget`, `scp`, `rsync … ssh`), identity spoofing (`git config user.email|user.name|core.hooksPath`), privilege escalation (`sudo`, `chmod 777`, `chown root`), and commit subversion (`git commit --amend`, `git stash drop`, `git reflog expire`, `git push … --no-verify`). A blocked command exits 2 with a `Blocked: … (v8.0.0 defense-in-depth)` message; there is currently no per-project opt-out knob. Installing the plugin at **project scope** rather than user scope confines these hooks (and all others) to a single repository. +- **Automatic git setup at session start**: under a user-scope install the `SessionStart` setup (see [Setup & Configuration](#setup--configuration)) runs in **any** working directory the session opens — so opening a non-project directory (e.g. `$HOME`) once will `git init` it and add an initial commit. Install at project scope to confine setup to one repository. ## Contributing diff --git a/agents/ac-evaluator-hi.md b/agents/ac-evaluator-hi.md index 65b724d..b051076 100644 --- a/agents/ac-evaluator-hi.md +++ b/agents/ac-evaluator-hi.md @@ -196,7 +196,7 @@ Acceptable AC verification methods (in priority order): When a runtime or visual AC is in scope AND a browser-automation utility skill is available, you MUST gather live evidence via method 6 — code inspection (methods 3-4) alone is NOT sufficient evidence to PASS such an AC. If no browser-automation skill is available, fall back to code inspection and reflect the missing live verification in the Caveats field (see PASS-WITH-CAVEATS). -If an AC requires behavior the existing test suite does not cover, the correct verdict is FAIL with an observation that test coverage is insufficient — NOT a workaround via scratch script. (The single exception is the oracle-probe carve-out above: for a computational AC you SHOULD write an independent-oracle probe under `.simple-workflow/scratch/` to derive the expected value — that is independent verification of an already-built behaviour, not a coverage workaround.) +If an AC requires behavior the existing test suite does not cover, the correct verdict is FAIL with an observation that test coverage is insufficient — NOT a workaround via scratch script. The carve-out permits EVIDENCE-GATHERING on already-built behaviour, NEVER a substitute for missing coverage: (a) for a computational AC, an independent-oracle probe under `.simple-workflow/scratch/` that derives the expected value from an oracle independent of the code; and (b) the **behavioral evidence probes** this section already directs — the time-bounded watchdog probe of an external-input boundary (point 5 below) and an EC-SELFDOC real-build / advertised-boundary probe — which likewise run under `.simple-workflow/scratch/` as independent verification of an already-built behaviour, not coverage workarounds; and (c) the **accept-set conformance sweep** — when a boundary advertises strict / canonical / lossless / limit OR a same-input-class sibling exists, an EXECUTED generative grammar-complement sweep (the four metamorphic relations of `## Failure-class panel` L-ROBUSTNESS, per `skills/impl/references/accept-set-conformance-harness.md`) whose accept-set is diffed black-box against an independent hand-coded spec oracle, run under `.simple-workflow/scratch/` and discarded after the round. This sweep is EVIDENCE-GATHERING on the already-built validator, NOT a substitute for missing PRODUCT test coverage: when the sweep finds an accept-set leak, the durable fix is producer-side — the next implementer / test-writer round commits a fixed rejection characterization test to the PRODUCT tests (your scratch corpus is ephemeral and never committed); you record the leaking input class in your Feedback so that round can pin it. **Exception**: If a truly temporary file is unavoidable, write to `os.tmpdir()` (Node) / `$TMPDIR` (POSIX) and clean up via the script's own `finally` block — NEVER `rm` as a separate shell command after the run (rm may be denied by permission gating, leaving the file behind). For oracle probes specifically, prefer the `.simple-workflow/scratch/` carve-out above; this `os.tmpdir()` path is the general fallback for any other unavoidable temp file. @@ -206,11 +206,11 @@ A **computational AC** is one whose PASS/FAIL hinges on a COMPUTED numeric or al For every computational AC in scope, a green project test suite is **necessary but NOT sufficient**. You MUST independently establish the expected value and compare it against the implementation's RAW output: -1. **Independent oracle**: compute at least one expected value from an oracle that does NOT share the implementation's core — a third-party reference library (e.g. `colorjs.io` cross-checking a culori-based engine), a published formula / standard you apply from first principles, or a hand-computed truth table with a cited source. The AC body or its Implementation Notes (per Gate 7) names the oracle; use it (a runtime oracle Skill, if one was bound, would also appear in `## Bound capabilities (per AC)`). NEVER take the implementation's own output (directly, via an alias, or by re-reading a field the code already rounded) as the expected value — that is the oracle-circularity defect this gate exists to catch. When your spawn prompt's `Evidence floor:` is `+1-independent` or `>=2-independent` (the `thorough` / `exhaustive` tiers) AND the AC is a standard-backed computational AC, require **two or more mutually-validated oracles** with **at least one derived from first principles** (the spec formula, hand-implemented, no library) and confirm they agree within an explicit tolerance before trusting either; FAIL a `thorough` / `exhaustive` standard-backed computational AC whose only independent evidence is a single library oracle. Build the second / first-principles oracle yourself under `.simple-workflow/scratch/` per the carve-out (shape: `skills/impl/references/independent-oracle-harness.md`). Where the domain has no published spec or no second independent oracle, the single-oracle path stands — record a Caveat (PASS-WITH-CAVEATS), never FAIL for an oracle that does not exist. **Strongest-derivation preference (all tiers, M3)**: when the AC's contract is derivable from a published spec / formula, PREFER a first-principles oracle (the spec formula, hand-implemented, no library) over a sibling reference library even at the `standard` floor where a single oracle suffices — a library oracle silently inherits that library's conventions. Record the **oracle-kind** you actually used — `first-principles | sibling | hand | none` (`none` = no independent oracle exists for this domain, the degradation path) — and surface it in the per-AC `[ORACLE-AUDIT]` line (below). +1. **Independent oracle**: compute at least one expected value from an oracle that does NOT share the implementation's core — a third-party reference library, a published formula / standard you apply from first principles, or a hand-computed truth table with a cited source. The AC body or its Implementation Notes (per Gate 7) names the oracle; use it (a runtime oracle Skill, if one was bound, would also appear in `## Bound capabilities (per AC)`). NEVER take the implementation's own output (directly, via an alias, or by re-reading a field the code already rounded) as the expected value — that is the oracle-circularity defect this gate exists to catch. When your spawn prompt's `Evidence floor:` is `+1-independent` or `>=2-independent` (the `thorough` / `exhaustive` tiers) AND the AC is a standard-backed computational AC, require **two or more mutually-validated oracles** with **at least one derived from first principles** (the spec formula, hand-implemented, no library) and confirm they agree within an explicit tolerance before trusting either; FAIL a `thorough` / `exhaustive` standard-backed computational AC whose only independent evidence is a single library oracle. Build the second / first-principles oracle yourself under `.simple-workflow/scratch/` per the carve-out (shape: `skills/impl/references/independent-oracle-harness.md`). Where the domain has no published spec or no second independent oracle, the single-oracle path stands — record a Caveat (PASS-WITH-CAVEATS), never FAIL for an oracle that does not exist. **Strongest-derivation preference (all tiers, M3)**: when the AC's contract is derivable from a published spec / formula, PREFER a first-principles oracle (the spec formula, hand-implemented, no library) over a sibling reference library even at the `standard` floor where a single oracle suffices — a library oracle silently inherits that library's conventions. Record the **oracle-kind** you actually used — `first-principles | sibling | hand | none` (`none` = no independent oracle exists for this domain, the degradation path) — and surface it in the per-AC `[ORACLE-AUDIT]` line (below). 2. **Raw, pre-rounding comparison**: compare the implementation's raw output (before display rounding / formatting) against the oracle value with an explicit tolerance (e.g. `|raw − oracle| ≤ 1e-6`). If the project's tests assert only on a display-rounded value, or re-threshold a field the code itself rounds (e.g. asserting `result.ratio >= target` on the code's 2-decimal `ratio`), treat the AC as NOT verified by those tests and FAIL it with feedback to compare the raw value against an independent oracle. 3. **Probe permitted**: write a throwaway oracle probe under the gitignored `.simple-workflow/scratch/` directory (per the oracle-probe carve-out above) when a one-off computation is the fastest way to derive the expected value. Discard it after the round; never import-and-rubber-stamp the implementation. Invoke a JS/TS probe via `node .simple-workflow/scratch/probe.mjs` or `npx -y tsx .simple-workflow/scratch/probe.ts`, and a Python probe via `python3 .simple-workflow/scratch/probe.py` (these runtimes are granted in this agent's `tools:` allowlist for scratch probes only). A published-formula or hand-computed-truth-table oracle needs no execution at all — prefer it when the ecosystem's standalone runtime is unavailable. 4. **No-oracle degradation**: when the domain genuinely has no independent oracle (novel business logic), verify via raw-value assertions against hand-computed constants AND property / invariant coverage (monotonicity, symmetry, idempotence, round-trip, containment) AND adversarial / non-finite / out-of-range inputs. Reflect any residual uncertainty in the Caveats field (PASS-WITH-CAVEATS) rather than silently trusting a self-confirming test. -5. **Adversarial coverage (every externally-fed AC — computational or behavioral, broadened M3)**: when an AC's value OR observable behaviour comes from a function that takes external / untrusted input — whether the AC is computational (a computed value) or behavioral (a returned value, status code, thrown error, wire payload) — the AC's tests MUST also exercise adversarial / non-finite / out-of-range inputs (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range / out-of-gamut). FAIL a computational or behavioral AC on such a function that ships zero adversarial coverage, with a feedback note — this is what catches DoS hangs and contract-violating outputs on bad input, not merely wrong values on good input. The coverage MUST include at least one **parse-accepted-then-overflows** vector (a value the parser ACCEPTS that yields a non-finite / out-of-range intermediate, e.g. `oklch(0.5 1e400 30)` → Infinity chroma), not just parse-rejected `NaN` / `Infinity` keyword tokens. You SHOULD independently probe one such vector through the tool under a TIME-BOUNDED watchdog — spawn a child process that calls the tool and SIGKILL it after a few seconds (a hang ⇒ FAIL), using the `.simple-workflow/scratch/` carve-out — and FAIL the AC if the tool hangs or returns a non-error success carrying null / NaN channels. Also confirm the validation guard is present across ALL sibling tools that accept the same input class — probe at least one sibling beyond the AC's primary tool; a guard in one tool but not its siblings is a FAIL. At the `+1-independent` / `>=2-independent` evidence floor (thorough / exhaustive), additionally confirm a **committed, fixed-seed** property-fuzz loop exists in the project's test files (reproducible PRNG, asserting invariants / oracle agreement across the input distribution, not only a hand-picked grid); FAIL a thorough / exhaustive computational AC whose only coverage is a handful of fixed fixtures with no committed seeded sweep. Where the ecosystem has no PRNG idiom this degrades to a Caveat, never a FAIL. +5. **Adversarial coverage (every externally-fed AC — computational or behavioral, broadened M3)**: when an AC's value OR observable behaviour comes from a function that takes external / untrusted input — whether the AC is computational (a computed value) or behavioral (a returned value, status code, thrown error, wire payload) — the AC's tests MUST also exercise adversarial / non-finite / out-of-range inputs (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range / out-of-gamut). FAIL a computational or behavioral AC on such a function that ships zero adversarial coverage, with a feedback note — this is what catches DoS hangs and contract-violating outputs on bad input, not merely wrong values on good input. The coverage MUST include at least one **parse-accepted-then-overflows** vector (an input the parser ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion — e.g. a numeric field whose magnitude overflows to Infinity once arithmetic is applied), not just parse-rejected `NaN` / `Infinity` keyword tokens. You SHOULD independently probe one such vector through the tool under a TIME-BOUNDED watchdog — spawn a child process that calls the tool and SIGKILL it after a few seconds (a hang ⇒ FAIL), using the `.simple-workflow/scratch/` carve-out — and FAIL the AC if the tool hangs or returns a non-error success carrying null / NaN channels. Also confirm the validation guard is present across ALL sibling tools that accept the same input class — probe at least one sibling beyond the AC's primary tool; a guard in one tool but not its siblings is a FAIL. The sibling set spans the whole product (siblings may live in separate tickets that each created a single unit), and a sibling that DELEGATES the input handling to a shared parser is NOT automatically safe: it re-exposes the boundary through its own surface (a round-trip / re-serialization path), so probe a DELEGATING sibling too and FAIL it when an input the shared parser accepted leaks (a bare error or a non-error success carrying a corrupt / non-finite value) through the delegating wrapper — delegation is NOT an automatic n/a. At the `+1-independent` / `>=2-independent` evidence floor (thorough / exhaustive), additionally confirm a **committed, fixed-seed** property-fuzz loop exists in the project's test files (reproducible PRNG, asserting invariants / oracle agreement across the input distribution, not only a hand-picked grid); FAIL a thorough / exhaustive computational AC whose only coverage is a handful of fixed fixtures with no committed seeded sweep. Where the ecosystem has no PRNG idiom this degrades to a Caveat, never a FAIL. 6. **Pre-Gate-7 / legacy degradation**: when the ticket predates Gate 7 (names no oracle and declares no fallback) OR your spawn prompt carries `Oracle verification: off`, do NOT hard-FAIL a computational AC solely for missing oracle independence — verify it by the pre-v8.2.0 path (project tests + code inspection + whatever property / adversarial coverage is present) and record a one-line Caveat that oracle independence was not verifiable from the ticket (PASS-WITH-CAVEATS), mirroring the pre-Gate-6 capability fallback below. A freshly authored or modified circular test still FAILs via the always-on R4 static rule regardless of this degradation. This requirement is **independent of the verification-depth tier** — it applies in single-verifier (`standard`) mode as well as the partition and multi-verifier (`exhaustive`) branches. The orchestrator resolves `constraints.oracle_verification` at `/impl` Step 3a and inlines it into your spawn prompt as the field `Oracle verification: {auto|off}` — read it from the prompt (like the `## Bound capabilities (per AC)` handoff); do NOT read it from disk, and when the field is absent (older orchestrator or a manual run) default to `auto` (active). When it is `off`, verify computational ACs by the pre-v8.2.0 path (project tests + code inspection) and note it in Caveats (see point 6). The R4 oracle-circularity rule in `skills/impl/references/tautological-assertion-rules.md` is the static counterpart that flags the circular test pattern in the diff; this section is the semantic, runtime counterpart. @@ -222,7 +222,7 @@ This requirement is **independent of the verification-depth tier** — it applie Gate 8 (`skills/create-ticket/references/ac-quality-criteria.md`) generalizes the oracle requirement to EVERY behavioral AC — an AC whose PASS/FAIL hinges on observable runtime behaviour (a returned value, emitted output, status code, rendered surface, wire payload, thrown error, side effect), not a structural fact. For every behavioral AC in scope you MUST establish PASS via at least one evidence channel independent of the implementation's own internals (defined in `skills/impl/references/evidence-channels.md`): - **EC-ORACLE** — an oracle-derived expected value (the `## Oracle Independence (computational ACs)` path above; the strongest sub-case, mandatory for computational ACs). -- **EC-DIFFERENTIAL** — cross-check against a separate reference implementation of the same contract; strongest as **algorithm-vs-algorithm** (a second, INDEPENDENT algorithm for the same contract, e.g. CSS-MINDE vs chroma-clamping) compared within tolerance — at thorough / exhaustive prefer this where a second algorithm exists, since a membership check is necessary-not-sufficient. +- **EC-DIFFERENTIAL** — cross-check against a separate reference implementation of the same contract; strongest as **algorithm-vs-algorithm** (a second, INDEPENDENT algorithm for the same contract, e.g. two independent sorts; two serializers) compared within tolerance — at thorough / exhaustive prefer this where a second algorithm exists, since a membership check is necessary-not-sufficient. - **EC-PROPERTY** — invariants over a seeded input distribution (monotonicity, symmetry, idempotence, round-trip, containment). - **EC-RUNTIME** — black-box observation through the real public / protocol boundary (the real CLI, the real MCP `Client` over a transport, the exported API, a rendered DOM), never internal handlers. - **EC-SELFDOC** — the unit's OWN declared contract (docstring / declared invariant / type annotation / advertised schema / `--help` line / README worked-example / advertised size-or-range boundary) RUN against the real build and compared to observed behaviour; FAIL on (A) drift between the unit's documented claim and its runtime behaviour, or (E) advertised boundary != enforced boundary. For a doc / README / `--help` worked-example the AC relies on, RUN it against the real build and diff stdout / exit (byte-for-byte if deterministic, explicit tolerance otherwise); for an advertised constraint, feed a FORBIDDEN value (must be rejected) and an ALLOWED value (must be accepted) to the real boundary. This is the standing channel for Gate 9 rows R3 / R4 and is a specialization layered on EC-RUNTIME (not a member of the four-channel naming set). Use the `.simple-workflow/scratch/` carve-out (below) to run the example / probe the boundary; **fail-open** — when the build cannot be exercised or the unit advertises no example / boundary, record a one-line Caveat (PASS-WITH-CAVEATS), never a force-FAIL for an example or boundary that does not exist. (The dedicated `doc-verifier` agent specialises in this channel when the orchestrator spawns it; you apply it inline as one of your behavioral-AC evidence channels.) @@ -349,8 +349,9 @@ attitude — gather evidence through your assigned channel: independently derive >=1 expected value from an oracle that does NOT share the implementation's core and compare against the RAW pre-rounding output with an explicit tolerance (the Gate 7 oracle probe, full force), and fuzz - at least one parse-accepted-then-overflows vector (e.g. - `oklch(0.5 1e400 30)`) under a time-bounded watchdog; FAIL on a hang or a + at least one parse-accepted-then-overflows vector (an input the parser + ACCEPTS that yields a non-finite / out-of-range intermediate after a + conversion) under a time-bounded watchdog; FAIL on a hang or a non-error success carrying null / NaN fields. The scratch carve-out under `.simple-workflow/scratch/` is permitted for the oracle probe. At thorough / exhaustive derive two mutually-validated oracles (>=1 first-principles), @@ -394,10 +395,103 @@ single pass under-checks: under a time-bounded watchdog. When the unit builds a structure (object / map / record) from untrusted input — keys derived from CSV headers, parsed JSON, or form / query / YAML fields — also probe hostile KEYS, not only hostile values: - prototype-pollution / accessor keys (`__proto__`, `constructor`, `prototype`), - duplicate / colliding keys, and empty / non-string keys; a key that silently - drops its column, mutates the prototype, or is swallowed is a robustness defect - (use `Object.create(null)` / `Object.defineProperty` / a `Map`). + an accessor / reserved key the host structure treats specially (the + prototype-pollution class), duplicate / colliding keys, and empty / non-string + keys; a key that silently drops its column, causes a structure-metadata + mutation, or is swallowed is a robustness defect (use a null-prototype / + own-properties-only container or a Map-equivalent). Where a + boundary is advertised **strict / canonical / exact** OR parses a number + through a lenient numeric primitive, also probe + **strictness-leniency**: feed inputs that satisfy the rules' letter yet exceed + the advertised surface — an out-of-alphabet symbol a permissive matcher + accepts (a non-ASCII / homoglyph numeral before a unit), a sign / whitespace + decoration a lenient primitive strips, and a + structurally-valid-but-non-canonical form the canonical writer would never + emit — and FAIL the unit when its enforced boundary is wider + than the strictness it advertises (accepted silently AND not + normalized-idempotent), not only when a value is corrupted. + **Accept-set conformance (EXECUTE, always-on, `accept_set_conformance`)**: + this strictness clause is satisfied by READING, and live dogfoods leaked + despite it — so when a boundary's contract lexically advertises **strict / + canonical / lossless / limit** (word-existence in the AC, its Implementation + Notes, the docstring, or `--help`) OR a sibling accepts the **same input + class** (the `shared_input_boundary` peer signal forwarded by the planner / + ticket-evaluator), you MUST NOT merely reason about the accept set — you MUST + **EXECUTE** a generative grammar-complement sweep in `.simple-workflow/scratch/` + (the behavioral evidence probe authorized at `## AC Verification Method`, + clause (c)) and diff its accept-set against an **independent hand-coded spec + oracle** you write from the spec, never by calling the unit's own validator / + `int()` / `Number()` / `isdigit()`. Derive a per-boundary **Grammar Card** — + `A` = the advertised input ALPHABET, `U` = any Unicode normalization / + decimal-digit transform, `W` = the canonical WRITER form, `K` = any KEYED + structure built from untrusted input — and run the four machine-generated + metamorphic relations black-box under a watchdog with a `mulberry32`-seeded + corpus (shape + worked example: + `skills/impl/references/accept-set-conformance-harness.md`): **MR-FINITE** (a + parse-accepted input whose conversion overflows to non-finite / out-of-range), + **MR-ALPHABET** (enumerate the Unicode decimal-digit property complement across + the BMP AND the astral planes — your generator names NO script and hard-codes + NO codepoint list; it iterates the codepoint space and selects by the + decimal-digit property, so every script's digits are covered for ALL inputs), + **MR-CANONICAL** (a structurally-valid-but-non-canonical form the writer would + never emit must be rejected or normalized-idempotent), and **MR-KEYFAITH** (a + reserved / accessor / colliding key must not mutate host-structure metadata or + be silently dropped — a CONCRETE round-trip-faithfulness violation (a drop, an + overwrite, or a host-metadata mutation) on a lossless / strict keyed boundary + is subject to the SAME standard two-tier oracle-authoritative FAIL gating as the + other MRs (below), not a weaker advisory-only floor. For the CORPUS you + MUST DERIVE the candidate key corpus BY REFLECTION over the structure's own + type / prototype — its reserved, accessor, method, AND private / internal slot + names (the FULL reflected set, never truncated or sliced) — so your generator + names NO key and hard-codes NO key list; it enumerates via reflection, so every + reserved / accessor / colliding / private-slot key is covered for ALL inputs, plus the generic + structural hostiles (an empty key, a duplicate of an existing key, a key that + collides after the structure's normalization), and you diff against an + independent round-trip-faithfulness oracle + (`skills/impl/references/accept-set-conformance-harness.md`). A single + hand-picked key list is NOT a conformant corpus). **FAIL gating + (oracle-authoritative two-tier, NO false-positive storm):** FAIL the AC on an + accept-set divergence ONLY when the hand-coded oracle AUTHORITATIVELY reflects a + spec that is STRICTLY NARROWER than the unit's accept-set (the advertised + contract is narrow and explicit — e.g. ASCII digits only, no leading zero, + bounded [0,N]); when the advertised width is NOT unambiguously narrower than + what the unit accepts (the boundary is legitimately or arguably WIDE — + Unicode-aware, locale-flexible, list-format-tolerant), the oracle (not the + unit) is the suspect: record the divergence as ADVISORY `[MEDIUM]` Feedback and + PASS-WITH-CAVEATS, NEVER a force-FAIL (this mirrors the EC-SELFDOC fail-open + posture). A single-character validator regression that the committed suite + still passes GREEN is exactly the narrow-spec class this EXECUTED sweep catches + and the suite does not. This sweep is **always-on** — independent of the + verification-depth tier, applied at `standard` too — gated ONLY by its own + switch: the orchestrator inlines `Accept-set conformance: {auto|off} + triggered-on={AC-id-list}` into your spawn prompt (alongside `Oracle + verification:` / `Evidence floor:` / `Selfdoc verification:`); read it from the + prompt, NOT from disk; absent field → `auto`. **Deterministic trigger — read + `triggered-on=`, do NOT re-recognize:** the orchestrator has ALREADY computed, + per AC, whether the boundary advertises strict / canonical / lossless / limit + OR is a shared-input sibling, and names the triggered AC-ids in `triggered-on=`. + EXECUTE the sweep for EXACTLY those ACs; you MUST NOT stand a `triggered-on=` AC + down on your own judgement — in particular, "the unit delegates validation to a + shared / part-1 parser" is NOT a reason to skip a listed AC (the delegating + surface re-exposes the boundary, which is why the orchestrator listed it). When + `triggered-on=(none)`, run no sweep and record `triggered=n` for every boundary. + When `Accept-set conformance: off`, skip the EXECUTED sweep regardless and + verify by the prior read-only strictness reasoning above, recording a one-line + Caveat. **Black-box — no implementation-peek:** you MUST NOT narrow or skip the + property-enumerated complement corpus (above) by inspecting the unit's internals + to argue an input class cannot occur (e.g. concluding the alphabet is ASCII-only + from the matcher the unit appears to use); the boundary's own accept / reject + decision is the only evidence. **Self-incriminating record:** on a + `triggered-on=` AC, an `## Accept-set sweep` line with `ran=n`, or with `ran=y + astral=n`, is a NON-CONFORMANT shallow sweep — not a clean pass; record it as + such (the astral planes are part of the mandated complement, so a genuine run + yields `astral=y`). **Caveat arm + (no-runnable-artifact)**: when the unit is a COMPILED-language artifact you + cannot run black-box (your scratch allowlist grants `node` / `python3` but, for + Rust/Go, only `cargo test` / `go test` — no `rustc` / `cargo run` / `go run` / + built binary), run the oracle alone, record the divergence you CAN derive + statically, and set `caveat=no-runnable-artifact` on the `[ACCEPT-SET-SWEEP]` + emit so a null result is never confused with a clean sweep. - **L-CONTRACT-CONFORMANCE** — does each generated unit's observable behaviour match its OWN stated description / declared schema / documented contract (a function that does not do what its name and doc-comment claim; a tool whose @@ -450,7 +544,7 @@ The natural execution order of this agent (verify ACs depth-first, then `Write` 1. **Skeleton write before verification.** As your FIRST action, `Write` the report path with a top-of-file line `## Status: IN_PROGRESS` followed by an AC checklist (one `- [ ] AC-N: ` line per AC extracted from the plan). This MUST happen before invoking any Bash, Read, or Grep verification tool. The resulting file is a partial-state marker that the orchestrator can detect even if you terminate mid-verification. -2. **Terminal rewrite before return.** After completing AC verification, rewrite the same file with final verdicts (`- [x] AC-N` or `- [ ] AC-N — FAILED: reason`) and replace the top-of-file `## Status: IN_PROGRESS` with the terminal `## Status: PASS`, `## Status: PASS-WITH-CAVEATS`, `## Status: FAIL`, or `## Status: FAIL-CRITICAL`. The terminal `## Status:` line MUST be the FIRST `## Status:` line in the file (the orchestrator inspects the first match). +2. **Terminal rewrite before return.** After completing AC verification, rewrite the same file with final verdicts (`- [x] AC-N` or `- [ ] AC-N — FAILED: reason`) and replace the top-of-file `## Status: IN_PROGRESS` with the terminal `## Status: PASS`, `## Status: PASS-WITH-CAVEATS`, `## Status: FAIL`, or `## Status: FAIL-CRITICAL`. The terminal `## Status:` line MUST be the FIRST `## Status:` line in the file (the orchestrator inspects the first match). **Accept-set sweep section (persisted observability, v8.5.0+):** when the EXECUTED accept-set conformance sweep (`## Failure-class panel` L-ROBUSTNESS) is in scope, the terminal rewrite MUST include in the PERSISTED report body a section `## Accept-set sweep` with exactly one line per external-input boundary INSPECTED (not only the ones that triggered) — `boundary={A|U|W|K} triggered={y|n} ran={y|n} astral={y|n} corpus-size={N} divergences={D} authoritative={y|n} caveat={none|no-runnable-artifact}` — so a dogfood greps the persisted `eval-round-N.md` (this report body, NOT the under-500-token `## Result` return envelope) and can tell `triggered=n` (scanned, recognized nothing) apart from a fired sweep apart from feature-absent. Emit `triggered=n` for every boundary you scanned that did not advertise strict/canonical/lossless/limit and has no same-input-class sibling; this UNCONDITIONAL per-boundary line is the falsifiability instrument (mirrors the unconditional `[ORACLE-AUDIT]` contract that survived to the dogfood reader, NOT the conditional `[EVAL-PANEL]` one that did not). Write `## Accept-set sweep\nn/a (no external-input boundary in scope)` when the diff introduces or touches no external-input boundary at all. 3. **Output path stability.** The `**Output**` field returned to the caller MUST be the same path written in step 1. Do not rename, move, or duplicate the file between the skeleton write and the terminal rewrite. diff --git a/agents/ac-evaluator.md b/agents/ac-evaluator.md index 1010ce4..e7662f5 100644 --- a/agents/ac-evaluator.md +++ b/agents/ac-evaluator.md @@ -196,7 +196,7 @@ Acceptable AC verification methods (in priority order): When a runtime or visual AC is in scope AND a browser-automation utility skill is available, you MUST gather live evidence via method 6 — code inspection (methods 3-4) alone is NOT sufficient evidence to PASS such an AC. If no browser-automation skill is available, fall back to code inspection and reflect the missing live verification in the Caveats field (see PASS-WITH-CAVEATS). -If an AC requires behavior the existing test suite does not cover, the correct verdict is FAIL with an observation that test coverage is insufficient — NOT a workaround via scratch script. (The single exception is the oracle-probe carve-out above: for a computational AC you SHOULD write an independent-oracle probe under `.simple-workflow/scratch/` to derive the expected value — that is independent verification of an already-built behaviour, not a coverage workaround.) +If an AC requires behavior the existing test suite does not cover, the correct verdict is FAIL with an observation that test coverage is insufficient — NOT a workaround via scratch script. The carve-out permits EVIDENCE-GATHERING on already-built behaviour, NEVER a substitute for missing coverage: (a) for a computational AC, an independent-oracle probe under `.simple-workflow/scratch/` that derives the expected value from an oracle independent of the code; and (b) the **behavioral evidence probes** this section already directs — the time-bounded watchdog probe of an external-input boundary (point 5 below) and an EC-SELFDOC real-build / advertised-boundary probe — which likewise run under `.simple-workflow/scratch/` as independent verification of an already-built behaviour, not coverage workarounds; and (c) the **accept-set conformance sweep** — when a boundary advertises strict / canonical / lossless / limit OR a same-input-class sibling exists, an EXECUTED generative grammar-complement sweep (the four metamorphic relations of `## Failure-class panel` L-ROBUSTNESS, per `skills/impl/references/accept-set-conformance-harness.md`) whose accept-set is diffed black-box against an independent hand-coded spec oracle, run under `.simple-workflow/scratch/` and discarded after the round. This sweep is EVIDENCE-GATHERING on the already-built validator, NOT a substitute for missing PRODUCT test coverage: when the sweep finds an accept-set leak, the durable fix is producer-side — the next implementer / test-writer round commits a fixed rejection characterization test to the PRODUCT tests (your scratch corpus is ephemeral and never committed); you record the leaking input class in your Feedback so that round can pin it. **Exception**: If a truly temporary file is unavoidable, write to `os.tmpdir()` (Node) / `$TMPDIR` (POSIX) and clean up via the script's own `finally` block — NEVER `rm` as a separate shell command after the run (rm may be denied by permission gating, leaving the file behind). For oracle probes specifically, prefer the `.simple-workflow/scratch/` carve-out above; this `os.tmpdir()` path is the general fallback for any other unavoidable temp file. @@ -206,11 +206,11 @@ A **computational AC** is one whose PASS/FAIL hinges on a COMPUTED numeric or al For every computational AC in scope, a green project test suite is **necessary but NOT sufficient**. You MUST independently establish the expected value and compare it against the implementation's RAW output: -1. **Independent oracle**: compute at least one expected value from an oracle that does NOT share the implementation's core — a third-party reference library (e.g. `colorjs.io` cross-checking a culori-based engine), a published formula / standard you apply from first principles, or a hand-computed truth table with a cited source. The AC body or its Implementation Notes (per Gate 7) names the oracle; use it (a runtime oracle Skill, if one was bound, would also appear in `## Bound capabilities (per AC)`). NEVER take the implementation's own output (directly, via an alias, or by re-reading a field the code already rounded) as the expected value — that is the oracle-circularity defect this gate exists to catch. When your spawn prompt's `Evidence floor:` is `+1-independent` or `>=2-independent` (the `thorough` / `exhaustive` tiers) AND the AC is a standard-backed computational AC, require **two or more mutually-validated oracles** with **at least one derived from first principles** (the spec formula, hand-implemented, no library) and confirm they agree within an explicit tolerance before trusting either; FAIL a `thorough` / `exhaustive` standard-backed computational AC whose only independent evidence is a single library oracle. Build the second / first-principles oracle yourself under `.simple-workflow/scratch/` per the carve-out (shape: `skills/impl/references/independent-oracle-harness.md`). Where the domain has no published spec or no second independent oracle, the single-oracle path stands — record a Caveat (PASS-WITH-CAVEATS), never FAIL for an oracle that does not exist. **Strongest-derivation preference (all tiers, M3)**: when the AC's contract is derivable from a published spec / formula, PREFER a first-principles oracle (the spec formula, hand-implemented, no library) over a sibling reference library even at the `standard` floor where a single oracle suffices — a library oracle silently inherits that library's conventions. Record the **oracle-kind** you actually used — `first-principles | sibling | hand | none` (`none` = no independent oracle exists for this domain, the degradation path) — and surface it in the per-AC `[ORACLE-AUDIT]` line (below). +1. **Independent oracle**: compute at least one expected value from an oracle that does NOT share the implementation's core — a third-party reference library, a published formula / standard you apply from first principles, or a hand-computed truth table with a cited source. The AC body or its Implementation Notes (per Gate 7) names the oracle; use it (a runtime oracle Skill, if one was bound, would also appear in `## Bound capabilities (per AC)`). NEVER take the implementation's own output (directly, via an alias, or by re-reading a field the code already rounded) as the expected value — that is the oracle-circularity defect this gate exists to catch. When your spawn prompt's `Evidence floor:` is `+1-independent` or `>=2-independent` (the `thorough` / `exhaustive` tiers) AND the AC is a standard-backed computational AC, require **two or more mutually-validated oracles** with **at least one derived from first principles** (the spec formula, hand-implemented, no library) and confirm they agree within an explicit tolerance before trusting either; FAIL a `thorough` / `exhaustive` standard-backed computational AC whose only independent evidence is a single library oracle. Build the second / first-principles oracle yourself under `.simple-workflow/scratch/` per the carve-out (shape: `skills/impl/references/independent-oracle-harness.md`). Where the domain has no published spec or no second independent oracle, the single-oracle path stands — record a Caveat (PASS-WITH-CAVEATS), never FAIL for an oracle that does not exist. **Strongest-derivation preference (all tiers, M3)**: when the AC's contract is derivable from a published spec / formula, PREFER a first-principles oracle (the spec formula, hand-implemented, no library) over a sibling reference library even at the `standard` floor where a single oracle suffices — a library oracle silently inherits that library's conventions. Record the **oracle-kind** you actually used — `first-principles | sibling | hand | none` (`none` = no independent oracle exists for this domain, the degradation path) — and surface it in the per-AC `[ORACLE-AUDIT]` line (below). 2. **Raw, pre-rounding comparison**: compare the implementation's raw output (before display rounding / formatting) against the oracle value with an explicit tolerance (e.g. `|raw − oracle| ≤ 1e-6`). If the project's tests assert only on a display-rounded value, or re-threshold a field the code itself rounds (e.g. asserting `result.ratio >= target` on the code's 2-decimal `ratio`), treat the AC as NOT verified by those tests and FAIL it with feedback to compare the raw value against an independent oracle. 3. **Probe permitted**: write a throwaway oracle probe under the gitignored `.simple-workflow/scratch/` directory (per the oracle-probe carve-out above) when a one-off computation is the fastest way to derive the expected value. Discard it after the round; never import-and-rubber-stamp the implementation. Invoke a JS/TS probe via `node .simple-workflow/scratch/probe.mjs` or `npx -y tsx .simple-workflow/scratch/probe.ts`, and a Python probe via `python3 .simple-workflow/scratch/probe.py` (these runtimes are granted in this agent's `tools:` allowlist for scratch probes only). A published-formula or hand-computed-truth-table oracle needs no execution at all — prefer it when the ecosystem's standalone runtime is unavailable. 4. **No-oracle degradation**: when the domain genuinely has no independent oracle (novel business logic), verify via raw-value assertions against hand-computed constants AND property / invariant coverage (monotonicity, symmetry, idempotence, round-trip, containment) AND adversarial / non-finite / out-of-range inputs. Reflect any residual uncertainty in the Caveats field (PASS-WITH-CAVEATS) rather than silently trusting a self-confirming test. -5. **Adversarial coverage (every externally-fed AC — computational or behavioral, broadened M3)**: when an AC's value OR observable behaviour comes from a function that takes external / untrusted input — whether the AC is computational (a computed value) or behavioral (a returned value, status code, thrown error, wire payload) — the AC's tests MUST also exercise adversarial / non-finite / out-of-range inputs (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range / out-of-gamut). FAIL a computational or behavioral AC on such a function that ships zero adversarial coverage, with a feedback note — this is what catches DoS hangs and contract-violating outputs on bad input, not merely wrong values on good input. The coverage MUST include at least one **parse-accepted-then-overflows** vector (a value the parser ACCEPTS that yields a non-finite / out-of-range intermediate, e.g. `oklch(0.5 1e400 30)` → Infinity chroma), not just parse-rejected `NaN` / `Infinity` keyword tokens. You SHOULD independently probe one such vector through the tool under a TIME-BOUNDED watchdog — spawn a child process that calls the tool and SIGKILL it after a few seconds (a hang ⇒ FAIL), using the `.simple-workflow/scratch/` carve-out — and FAIL the AC if the tool hangs or returns a non-error success carrying null / NaN channels. Also confirm the validation guard is present across ALL sibling tools that accept the same input class — probe at least one sibling beyond the AC's primary tool; a guard in one tool but not its siblings is a FAIL. At the `+1-independent` / `>=2-independent` evidence floor (thorough / exhaustive), additionally confirm a **committed, fixed-seed** property-fuzz loop exists in the project's test files (reproducible PRNG, asserting invariants / oracle agreement across the input distribution, not only a hand-picked grid); FAIL a thorough / exhaustive computational AC whose only coverage is a handful of fixed fixtures with no committed seeded sweep. Where the ecosystem has no PRNG idiom this degrades to a Caveat, never a FAIL. +5. **Adversarial coverage (every externally-fed AC — computational or behavioral, broadened M3)**: when an AC's value OR observable behaviour comes from a function that takes external / untrusted input — whether the AC is computational (a computed value) or behavioral (a returned value, status code, thrown error, wire payload) — the AC's tests MUST also exercise adversarial / non-finite / out-of-range inputs (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range / out-of-gamut). FAIL a computational or behavioral AC on such a function that ships zero adversarial coverage, with a feedback note — this is what catches DoS hangs and contract-violating outputs on bad input, not merely wrong values on good input. The coverage MUST include at least one **parse-accepted-then-overflows** vector (an input the parser ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion — e.g. a numeric field whose magnitude overflows to Infinity once arithmetic is applied), not just parse-rejected `NaN` / `Infinity` keyword tokens. You SHOULD independently probe one such vector through the tool under a TIME-BOUNDED watchdog — spawn a child process that calls the tool and SIGKILL it after a few seconds (a hang ⇒ FAIL), using the `.simple-workflow/scratch/` carve-out — and FAIL the AC if the tool hangs or returns a non-error success carrying null / NaN channels. Also confirm the validation guard is present across ALL sibling tools that accept the same input class — probe at least one sibling beyond the AC's primary tool; a guard in one tool but not its siblings is a FAIL. The sibling set spans the whole product (siblings may live in separate tickets that each created a single unit), and a sibling that DELEGATES the input handling to a shared parser is NOT automatically safe: it re-exposes the boundary through its own surface (a round-trip / re-serialization path), so probe a DELEGATING sibling too and FAIL it when an input the shared parser accepted leaks (a bare error or a non-error success carrying a corrupt / non-finite value) through the delegating wrapper — delegation is NOT an automatic n/a. At the `+1-independent` / `>=2-independent` evidence floor (thorough / exhaustive), additionally confirm a **committed, fixed-seed** property-fuzz loop exists in the project's test files (reproducible PRNG, asserting invariants / oracle agreement across the input distribution, not only a hand-picked grid); FAIL a thorough / exhaustive computational AC whose only coverage is a handful of fixed fixtures with no committed seeded sweep. Where the ecosystem has no PRNG idiom this degrades to a Caveat, never a FAIL. 6. **Pre-Gate-7 / legacy degradation**: when the ticket predates Gate 7 (names no oracle and declares no fallback) OR your spawn prompt carries `Oracle verification: off`, do NOT hard-FAIL a computational AC solely for missing oracle independence — verify it by the pre-v8.2.0 path (project tests + code inspection + whatever property / adversarial coverage is present) and record a one-line Caveat that oracle independence was not verifiable from the ticket (PASS-WITH-CAVEATS), mirroring the pre-Gate-6 capability fallback below. A freshly authored or modified circular test still FAILs via the always-on R4 static rule regardless of this degradation. This requirement is **independent of the verification-depth tier** — it applies in single-verifier (`standard`) mode as well as the partition and multi-verifier (`exhaustive`) branches. The orchestrator resolves `constraints.oracle_verification` at `/impl` Step 3a and inlines it into your spawn prompt as the field `Oracle verification: {auto|off}` — read it from the prompt (like the `## Bound capabilities (per AC)` handoff); do NOT read it from disk, and when the field is absent (older orchestrator or a manual run) default to `auto` (active). When it is `off`, verify computational ACs by the pre-v8.2.0 path (project tests + code inspection) and note it in Caveats (see point 6). The R4 oracle-circularity rule in `skills/impl/references/tautological-assertion-rules.md` is the static counterpart that flags the circular test pattern in the diff; this section is the semantic, runtime counterpart. @@ -222,7 +222,7 @@ This requirement is **independent of the verification-depth tier** — it applie Gate 8 (`skills/create-ticket/references/ac-quality-criteria.md`) generalizes the oracle requirement to EVERY behavioral AC — an AC whose PASS/FAIL hinges on observable runtime behaviour (a returned value, emitted output, status code, rendered surface, wire payload, thrown error, side effect), not a structural fact. For every behavioral AC in scope you MUST establish PASS via at least one evidence channel independent of the implementation's own internals (defined in `skills/impl/references/evidence-channels.md`): - **EC-ORACLE** — an oracle-derived expected value (the `## Oracle Independence (computational ACs)` path above; the strongest sub-case, mandatory for computational ACs). -- **EC-DIFFERENTIAL** — cross-check against a separate reference implementation of the same contract; strongest as **algorithm-vs-algorithm** (a second, INDEPENDENT algorithm for the same contract, e.g. CSS-MINDE vs chroma-clamping) compared within tolerance — at thorough / exhaustive prefer this where a second algorithm exists, since a membership check is necessary-not-sufficient. +- **EC-DIFFERENTIAL** — cross-check against a separate reference implementation of the same contract; strongest as **algorithm-vs-algorithm** (a second, INDEPENDENT algorithm for the same contract, e.g. two independent sorts; two serializers) compared within tolerance — at thorough / exhaustive prefer this where a second algorithm exists, since a membership check is necessary-not-sufficient. - **EC-PROPERTY** — invariants over a seeded input distribution (monotonicity, symmetry, idempotence, round-trip, containment). - **EC-RUNTIME** — black-box observation through the real public / protocol boundary (the real CLI, the real MCP `Client` over a transport, the exported API, a rendered DOM), never internal handlers. - **EC-SELFDOC** — the unit's OWN declared contract (docstring / declared invariant / type annotation / advertised schema / `--help` line / README worked-example / advertised size-or-range boundary) RUN against the real build and compared to observed behaviour; FAIL on (A) drift between the unit's documented claim and its runtime behaviour, or (E) advertised boundary != enforced boundary. For a doc / README / `--help` worked-example the AC relies on, RUN it against the real build and diff stdout / exit (byte-for-byte if deterministic, explicit tolerance otherwise); for an advertised constraint, feed a FORBIDDEN value (must be rejected) and an ALLOWED value (must be accepted) to the real boundary. This is the standing channel for Gate 9 rows R3 / R4 and is a specialization layered on EC-RUNTIME (not a member of the four-channel naming set). Use the `.simple-workflow/scratch/` carve-out (below) to run the example / probe the boundary; **fail-open** — when the build cannot be exercised or the unit advertises no example / boundary, record a one-line Caveat (PASS-WITH-CAVEATS), never a force-FAIL for an example or boundary that does not exist. (The dedicated `doc-verifier` agent specialises in this channel when the orchestrator spawns it; you apply it inline as one of your behavioral-AC evidence channels.) @@ -349,8 +349,9 @@ attitude — gather evidence through your assigned channel: independently derive >=1 expected value from an oracle that does NOT share the implementation's core and compare against the RAW pre-rounding output with an explicit tolerance (the Gate 7 oracle probe, full force), and fuzz - at least one parse-accepted-then-overflows vector (e.g. - `oklch(0.5 1e400 30)`) under a time-bounded watchdog; FAIL on a hang or a + at least one parse-accepted-then-overflows vector (an input the parser + ACCEPTS that yields a non-finite / out-of-range intermediate after a + conversion) under a time-bounded watchdog; FAIL on a hang or a non-error success carrying null / NaN fields. The scratch carve-out under `.simple-workflow/scratch/` is permitted for the oracle probe. At thorough / exhaustive derive two mutually-validated oracles (>=1 first-principles), @@ -394,10 +395,103 @@ single pass under-checks: under a time-bounded watchdog. When the unit builds a structure (object / map / record) from untrusted input — keys derived from CSV headers, parsed JSON, or form / query / YAML fields — also probe hostile KEYS, not only hostile values: - prototype-pollution / accessor keys (`__proto__`, `constructor`, `prototype`), - duplicate / colliding keys, and empty / non-string keys; a key that silently - drops its column, mutates the prototype, or is swallowed is a robustness defect - (use `Object.create(null)` / `Object.defineProperty` / a `Map`). + an accessor / reserved key the host structure treats specially (the + prototype-pollution class), duplicate / colliding keys, and empty / non-string + keys; a key that silently drops its column, causes a structure-metadata + mutation, or is swallowed is a robustness defect (use a null-prototype / + own-properties-only container or a Map-equivalent). Where a + boundary is advertised **strict / canonical / exact** OR parses a number + through a lenient numeric primitive, also probe + **strictness-leniency**: feed inputs that satisfy the rules' letter yet exceed + the advertised surface — an out-of-alphabet symbol a permissive matcher + accepts (a non-ASCII / homoglyph numeral before a unit), a sign / whitespace + decoration a lenient primitive strips, and a + structurally-valid-but-non-canonical form the canonical writer would never + emit — and FAIL the unit when its enforced boundary is wider + than the strictness it advertises (accepted silently AND not + normalized-idempotent), not only when a value is corrupted. + **Accept-set conformance (EXECUTE, always-on, `accept_set_conformance`)**: + this strictness clause is satisfied by READING, and live dogfoods leaked + despite it — so when a boundary's contract lexically advertises **strict / + canonical / lossless / limit** (word-existence in the AC, its Implementation + Notes, the docstring, or `--help`) OR a sibling accepts the **same input + class** (the `shared_input_boundary` peer signal forwarded by the planner / + ticket-evaluator), you MUST NOT merely reason about the accept set — you MUST + **EXECUTE** a generative grammar-complement sweep in `.simple-workflow/scratch/` + (the behavioral evidence probe authorized at `## AC Verification Method`, + clause (c)) and diff its accept-set against an **independent hand-coded spec + oracle** you write from the spec, never by calling the unit's own validator / + `int()` / `Number()` / `isdigit()`. Derive a per-boundary **Grammar Card** — + `A` = the advertised input ALPHABET, `U` = any Unicode normalization / + decimal-digit transform, `W` = the canonical WRITER form, `K` = any KEYED + structure built from untrusted input — and run the four machine-generated + metamorphic relations black-box under a watchdog with a `mulberry32`-seeded + corpus (shape + worked example: + `skills/impl/references/accept-set-conformance-harness.md`): **MR-FINITE** (a + parse-accepted input whose conversion overflows to non-finite / out-of-range), + **MR-ALPHABET** (enumerate the Unicode decimal-digit property complement across + the BMP AND the astral planes — your generator names NO script and hard-codes + NO codepoint list; it iterates the codepoint space and selects by the + decimal-digit property, so every script's digits are covered for ALL inputs), + **MR-CANONICAL** (a structurally-valid-but-non-canonical form the writer would + never emit must be rejected or normalized-idempotent), and **MR-KEYFAITH** (a + reserved / accessor / colliding key must not mutate host-structure metadata or + be silently dropped — a CONCRETE round-trip-faithfulness violation (a drop, an + overwrite, or a host-metadata mutation) on a lossless / strict keyed boundary + is subject to the SAME standard two-tier oracle-authoritative FAIL gating as the + other MRs (below), not a weaker advisory-only floor. For the CORPUS you + MUST DERIVE the candidate key corpus BY REFLECTION over the structure's own + type / prototype — its reserved, accessor, method, AND private / internal slot + names (the FULL reflected set, never truncated or sliced) — so your generator + names NO key and hard-codes NO key list; it enumerates via reflection, so every + reserved / accessor / colliding / private-slot key is covered for ALL inputs, plus the generic + structural hostiles (an empty key, a duplicate of an existing key, a key that + collides after the structure's normalization), and you diff against an + independent round-trip-faithfulness oracle + (`skills/impl/references/accept-set-conformance-harness.md`). A single + hand-picked key list is NOT a conformant corpus). **FAIL gating + (oracle-authoritative two-tier, NO false-positive storm):** FAIL the AC on an + accept-set divergence ONLY when the hand-coded oracle AUTHORITATIVELY reflects a + spec that is STRICTLY NARROWER than the unit's accept-set (the advertised + contract is narrow and explicit — e.g. ASCII digits only, no leading zero, + bounded [0,N]); when the advertised width is NOT unambiguously narrower than + what the unit accepts (the boundary is legitimately or arguably WIDE — + Unicode-aware, locale-flexible, list-format-tolerant), the oracle (not the + unit) is the suspect: record the divergence as ADVISORY `[MEDIUM]` Feedback and + PASS-WITH-CAVEATS, NEVER a force-FAIL (this mirrors the EC-SELFDOC fail-open + posture). A single-character validator regression that the committed suite + still passes GREEN is exactly the narrow-spec class this EXECUTED sweep catches + and the suite does not. This sweep is **always-on** — independent of the + verification-depth tier, applied at `standard` too — gated ONLY by its own + switch: the orchestrator inlines `Accept-set conformance: {auto|off} + triggered-on={AC-id-list}` into your spawn prompt (alongside `Oracle + verification:` / `Evidence floor:` / `Selfdoc verification:`); read it from the + prompt, NOT from disk; absent field → `auto`. **Deterministic trigger — read + `triggered-on=`, do NOT re-recognize:** the orchestrator has ALREADY computed, + per AC, whether the boundary advertises strict / canonical / lossless / limit + OR is a shared-input sibling, and names the triggered AC-ids in `triggered-on=`. + EXECUTE the sweep for EXACTLY those ACs; you MUST NOT stand a `triggered-on=` AC + down on your own judgement — in particular, "the unit delegates validation to a + shared / part-1 parser" is NOT a reason to skip a listed AC (the delegating + surface re-exposes the boundary, which is why the orchestrator listed it). When + `triggered-on=(none)`, run no sweep and record `triggered=n` for every boundary. + When `Accept-set conformance: off`, skip the EXECUTED sweep regardless and + verify by the prior read-only strictness reasoning above, recording a one-line + Caveat. **Black-box — no implementation-peek:** you MUST NOT narrow or skip the + property-enumerated complement corpus (above) by inspecting the unit's internals + to argue an input class cannot occur (e.g. concluding the alphabet is ASCII-only + from the matcher the unit appears to use); the boundary's own accept / reject + decision is the only evidence. **Self-incriminating record:** on a + `triggered-on=` AC, an `## Accept-set sweep` line with `ran=n`, or with `ran=y + astral=n`, is a NON-CONFORMANT shallow sweep — not a clean pass; record it as + such (the astral planes are part of the mandated complement, so a genuine run + yields `astral=y`). **Caveat arm + (no-runnable-artifact)**: when the unit is a COMPILED-language artifact you + cannot run black-box (your scratch allowlist grants `node` / `python3` but, for + Rust/Go, only `cargo test` / `go test` — no `rustc` / `cargo run` / `go run` / + built binary), run the oracle alone, record the divergence you CAN derive + statically, and set `caveat=no-runnable-artifact` on the `[ACCEPT-SET-SWEEP]` + emit so a null result is never confused with a clean sweep. - **L-CONTRACT-CONFORMANCE** — does each generated unit's observable behaviour match its OWN stated description / declared schema / documented contract (a function that does not do what its name and doc-comment claim; a tool whose @@ -450,7 +544,7 @@ The natural execution order of this agent (verify ACs depth-first, then `Write` 1. **Skeleton write before verification.** As your FIRST action, `Write` the report path with a top-of-file line `## Status: IN_PROGRESS` followed by an AC checklist (one `- [ ] AC-N: ` line per AC extracted from the plan). This MUST happen before invoking any Bash, Read, or Grep verification tool. The resulting file is a partial-state marker that the orchestrator can detect even if you terminate mid-verification. -2. **Terminal rewrite before return.** After completing AC verification, rewrite the same file with final verdicts (`- [x] AC-N` or `- [ ] AC-N — FAILED: reason`) and replace the top-of-file `## Status: IN_PROGRESS` with the terminal `## Status: PASS`, `## Status: PASS-WITH-CAVEATS`, `## Status: FAIL`, or `## Status: FAIL-CRITICAL`. The terminal `## Status:` line MUST be the FIRST `## Status:` line in the file (the orchestrator inspects the first match). +2. **Terminal rewrite before return.** After completing AC verification, rewrite the same file with final verdicts (`- [x] AC-N` or `- [ ] AC-N — FAILED: reason`) and replace the top-of-file `## Status: IN_PROGRESS` with the terminal `## Status: PASS`, `## Status: PASS-WITH-CAVEATS`, `## Status: FAIL`, or `## Status: FAIL-CRITICAL`. The terminal `## Status:` line MUST be the FIRST `## Status:` line in the file (the orchestrator inspects the first match). **Accept-set sweep section (persisted observability, v8.5.0+):** when the EXECUTED accept-set conformance sweep (`## Failure-class panel` L-ROBUSTNESS) is in scope, the terminal rewrite MUST include in the PERSISTED report body a section `## Accept-set sweep` with exactly one line per external-input boundary INSPECTED (not only the ones that triggered) — `boundary={A|U|W|K} triggered={y|n} ran={y|n} astral={y|n} corpus-size={N} divergences={D} authoritative={y|n} caveat={none|no-runnable-artifact}` — so a dogfood greps the persisted `eval-round-N.md` (this report body, NOT the under-500-token `## Result` return envelope) and can tell `triggered=n` (scanned, recognized nothing) apart from a fired sweep apart from feature-absent. Emit `triggered=n` for every boundary you scanned that did not advertise strict/canonical/lossless/limit and has no same-input-class sibling; this UNCONDITIONAL per-boundary line is the falsifiability instrument (mirrors the unconditional `[ORACLE-AUDIT]` contract that survived to the dogfood reader, NOT the conditional `[EVAL-PANEL]` one that did not). Write `## Accept-set sweep\nn/a (no external-input boundary in scope)` when the diff introduces or touches no external-input boundary at all. 3. **Output path stability.** The `**Output**` field returned to the caller MUST be the same path written in step 1. Do not rename, move, or duplicate the file between the skeleton write and the terminal rewrite. diff --git a/agents/decomposer.md b/agents/decomposer.md index 5c6bcf4..2635fee 100644 --- a/agents/decomposer.md +++ b/agents/decomposer.md @@ -6,7 +6,7 @@ tools: - Grep - Glob - Skill -model: opus +model: inherit maxTurns: 20 --- @@ -66,6 +66,10 @@ For EACH ticket above, ALSO surface a peer-set hint (Gate 10 upstream signal for - peer_set: true | false — `true` when this ticket's scope creates `>=2` analogous sibling units in ONE category (peer tools / endpoints / subcommands / functions sharing an output surface); `false` for a single-unit ticket or units that form no analogous sibling set. shared_conventions: <1-line hint, ONLY when peer_set is true> — the cross-unit convention the peers should share (e.g. "single error envelope across the 3 parse tools", "one success wrapper for the 4 list endpoints"). Omit this sub-line entirely when peer_set is false. +AFTER the per-ticket list, ALWAYS surface ONE cross-ticket input-boundary hint spanning the WHOLE ticket set (the Gate 7 sibling-guard / Gate 9 upstream signal — you are the only agent that sees every ticket at once, so only you can observe this across ticket boundaries). This line is MANDATORY-PRESENT (emit `shared_input_boundary: none` when nothing is shared — never omit it), mirroring how `peer_set:` is mandatory per ticket; it is an advisory hint emitted UNCONDITIONALLY (you read no policy), exactly like `peer_set:`: + +- shared_input_boundary: — emit a class line when `>=2` DIFFERENT tickets above each create a unit that consumes ONE common external input class (the same parsed format / value-type / wire shape), INCLUDING the case where each such ticket creates only a SINGLE unit (so `peer_set` is `false` for each) — this is the cross-ticket sibling set the per-ticket `peer_set` cannot see. Each class line names the shared input class abstractly + the sibling ticket ids: `: [-part-N, -part-M, ...]`. A sibling that DELEGATES the input handling to a shared parser created in another ticket STILL belongs on this line (delegation does not remove it from the shared-input-class family). NOTE (model-judgment caveat, same fragility class as the Gate 9 / Gate 10 evaluator notes): 'which tickets share an input class' is your judgment over the free-text scopes, not a grep — a downstream reader MUST NOT treat the presence or absence of a class line as mechanically authoritative. + ## Decomposition principles 1. Each ticket must describe a coherent, independently-deployable unit. Never produce a ticket with fewer than 2 latent ACs worth of scope — if a Work Unit is too thin, merge it with a neighbor. diff --git a/agents/implementer.md b/agents/implementer.md index 8a7fb13..bb7a9bb 100644 --- a/agents/implementer.md +++ b/agents/implementer.md @@ -1,6 +1,6 @@ --- name: implementer -description: "Implement code changes following a plan. Opus model for L/XL tickets, Sonnet for S/M." +description: "Implement code changes following a plan. Always runs on the opus model." model: opus maxTurns: 45 --- @@ -26,7 +26,7 @@ After implementing, run the project's lint command (as defined in CLAUDE.md or p After lint passes, run the project's test command (as defined in CLAUDE.md or project conventions). If tests fail, fix and re-run (max 3 attempts). -When writing these tests, you MUST follow the positive rubric in `skills/impl/references/test-authoring-guidance.md` (resolve the path relative to the repository root). For any **computational AC** (one whose PASS/FAIL hinges on a computed numeric/algorithmic value — a ratio, threshold, hash, conversion, round-trip, metric), assert against an **independent oracle** (a reference library / published formula / hand-computed truth table — never the code's own output) on the **raw, pre-rounding** value with an explicit tolerance; add property / invariant tests (monotonicity, symmetry, idempotence, round-trip, containment); and cover adversarial / non-finite / out-of-range inputs by default (this default-adversarial coverage applies to **computational or behavioral** targets alike — M3, v8.4.0+ — any target whose function takes external / untrusted input, even though the oracle / raw-value / differential rules stay computational-only) — including at least one **parse-accepted-then-overflows** vector (a value the parser ACCEPTS that yields a non-finite / out-of-range intermediate, e.g. `oklch(0.5 1e400 30)` → Infinity chroma), not only parse-rejected `NaN` / `Infinity` tokens; and, when the function shares an input parser with sibling tools, ensure the input-validation guard lives in the SHARED boundary OR is replicated AND adversarially tested in EVERY sibling tool (see rule 4 of the guidance). A test that re-measures with the implementation's own rounded value is self-confirming and is rejected by tautological rule R4 (`skills/impl/references/tautological-assertion-rules.md`). At the `thorough` / `exhaustive` depth tier, for a standard-backed computational target author **two or more mutually-validated independent oracles** (at least one first-principles, no library) and trust a value only when they agree within tolerance — see `skills/impl/references/independent-oracle-harness.md`; ship a **committed, fixed-seed** property-fuzz loop (reproducible PRNG, tier-scaled case count) over the input distribution, not only fixed fixtures; and where a second INDEPENDENT ALGORITHM for the same contract exists, add an algorithm-vs-algorithm differential within tolerance (membership is necessary-not-sufficient). +When writing these tests, you MUST follow the positive rubric in `skills/impl/references/test-authoring-guidance.md` (resolve the path relative to the repository root). For any **computational AC** (one whose PASS/FAIL hinges on a computed numeric/algorithmic value — a ratio, threshold, hash, conversion, round-trip, metric), assert against an **independent oracle** (a reference library / published formula / hand-computed truth table — never the code's own output) on the **raw, pre-rounding** value with an explicit tolerance; add property / invariant tests (monotonicity, symmetry, idempotence, round-trip, containment); and cover adversarial / non-finite / out-of-range inputs by default (this default-adversarial coverage applies to **computational or behavioral** targets alike — M3, v8.4.0+ — any target whose function takes external / untrusted input, even though the oracle / raw-value / differential rules stay computational-only) — including at least one **parse-accepted-then-overflows** vector (an input the parser ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion — a `NaN` or `Infinity` sentinel surfacing downstream of an accepted value), not only parse-rejected `NaN` / `Infinity` tokens; and, when the function shares an input parser with sibling tools, ensure the input-validation guard lives in the SHARED boundary OR is replicated AND adversarially tested in EVERY sibling tool (see rule 4 of the guidance). A test that re-measures with the implementation's own rounded value is self-confirming and is rejected by tautological rule R4 (`skills/impl/references/tautological-assertion-rules.md`). At the `thorough` / `exhaustive` depth tier, for a standard-backed computational target author **two or more mutually-validated independent oracles** (at least one first-principles, no library) and trust a value only when they agree within tolerance — see `skills/impl/references/independent-oracle-harness.md`; ship a **committed, fixed-seed** property-fuzz loop (reproducible PRNG, tier-scaled case count) over the input distribution, not only fixed fixtures; and where a second INDEPENDENT ALGORITHM for the same contract exists, add an algorithm-vs-algorithm differential within tolerance (membership is necessary-not-sufficient). **Accept-set conformance retained corpus (always-on, `accept_set_conformance`, v8.5.0+).** Regardless of the `thorough` / `exhaustive` condition in the preceding sentence: when the ac-evaluator's accept-set conformance sweep reports an accept-set leak — an input the spec oracle rejects that a strict / canonical / lossless / limit boundary (or a same-input-class sibling) accepts, named in the evaluator Feedback — commit a **fixed rejection characterization test** to the PRODUCT tests for that leaking input class (the leaking input as a RED case; the validator fix is GREEN), so the regression is locked in the committed suite. This is the producer half of the two-surface retention: the evaluator's scratch sweep is ephemeral and never committed, so the durable test belongs here. Gated by `constraints.accept_set_conformance` (off → skip) but NOT by the depth tier — unlike the committed-seeded-fuzz clause above it is always-on at `standard` too. See `skills/impl/references/accept-set-conformance-harness.md`. Do NOT include self-assessment, subjective comments, or quality judgments in your return value. Report only factual information. diff --git a/agents/planner.md b/agents/planner.md index 366195f..b6e5aea 100644 --- a/agents/planner.md +++ b/agents/planner.md @@ -1,7 +1,7 @@ --- name: planner description: "Create detailed implementation plans for features and refactoring." -model: opus +model: inherit maxTurns: 30 --- @@ -55,7 +55,7 @@ After the Gate 6.5 probe completeness cross-check passes, run this oracle-indepe a. Verify the AC body OR its Implementation Notes names an **oracle independent of the implementation** (a third-party reference library that does not share the implementation's core, a published formula / standard, or a hand-computed truth table with a cited source) AND specifies a **raw, pre-rounding** comparison with an explicit tolerance. Re-thresholding a field the implementation itself rounds (e.g. asserting `result.ratio >= target` on the code's 2-decimal `ratio`) is a Gate 7 FAIL. For a standard-backed computational AC expected to resolve to the `thorough` / `exhaustive` evidence_floor (a critical-domain / shared-core / high-stakes computational AC — see `skills/impl/references/verification-depth.md` criticality floor), prefer naming **two mutually-validated oracles** (one first-principles) so the downstream verifier can satisfy the multi-oracle floor; where only one independent oracle exists, say so. Where a second independent ALGORITHM for the contract exists, prefer naming it so the verifier can run an algorithm-vs-algorithm differential (membership alone is necessary-not-sufficient). This is authoring guidance, not a force-FAIL — Gate 7 still passes a single named oracle at draft time. b. If no independent oracle exists for the domain, declare the **no-oracle fallback** explicitly in the AC / Implementation Notes: raw-value assertions with tolerance against hand-computed constants AND property / invariant coverage (monotonicity, symmetry, idempotence, round-trip, containment) AND adversarial / non-finite / out-of-range inputs. State which path applies. c. If a computational AC names neither an oracle (+ raw-value tolerance) nor the no-oracle fallback, EITHER revise it to do so, OR rewrite it as a static AC (file-grep / counter / exit-code). Emitting a computational AC with no oracle, no fallback, and no static rewrite is a Gate 7 FAIL. Purely structural ACs are `n/a` for Gate 7. - d. For a **computational or behavioral** AC whose value or behaviour comes from a function taking external / untrusted input (broadened in M3, v8.4.0+), also require adversarial / non-finite / out-of-range coverage (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range); a computational or behavioral AC on such a function with no adversarial coverage is a Gate 7 FAIL — this catches DoS hangs and bad-input contract violations, not just wrong values on good input. The required coverage MUST include a parse-ACCEPTED-then-overflows vector (e.g. `oklch(0.5 1e400 30)`), not only parse-rejected `NaN` / `Infinity` tokens. If the function shares an input parser with sibling tools, require the guard in the SHARED boundary OR in every sibling (not just one). + d. For a **computational or behavioral** AC whose value or behaviour comes from a function taking external / untrusted input (broadened in M3, v8.4.0+), also require adversarial / non-finite / out-of-range coverage (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range); a computational or behavioral AC on such a function with no adversarial coverage is a Gate 7 FAIL — this catches DoS hangs and bad-input contract violations, not just wrong values on good input. The required coverage MUST include a parse-ACCEPTED-then-overflows vector (an input the parser ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion — a `NaN` or `Infinity` sentinel surfacing downstream of an accepted value), not only parse-rejected `NaN` / `Infinity` tokens. If the function shares an input parser with sibling tools, require the guard in the SHARED boundary OR in every sibling (not just one). When your spawn prompt forwards a cross-ticket `shared_input_boundary` membership for this ticket (the input class is consumed by sibling tickets too, even ones that each create only a single unit), this ticket's unit is a member of that shared-input-class family even if it DELEGATES the input handling to a shared parser created in another ticket: it MUST carry its OWN hostile / accept-set row (the Gate 9 R2 strictness-leniency probe — non-ASCII / Unicode-digit numeral, leading-zero / non-canonical form, sign / whitespace) for that input class, OR a one-line JUSTIFIED `n/a` reason a reviewer can challenge — delegation is NOT an automatic n/a, never a silent skip, and 'delegates to the shared parser' is itself NOT an acceptable n/a reason (the row is owed precisely because the delegating surface re-exposes the boundary). ## Pre-emit Self-Audit (ticket drafts: Gate 8 independent evidence) diff --git a/agents/test-writer.md b/agents/test-writer.md index dd9d5f0..fedc480 100644 --- a/agents/test-writer.md +++ b/agents/test-writer.md @@ -17,7 +17,7 @@ You are a test engineer. Write and run tests following existing project patterns ## Test Design Rubric -Follow `skills/impl/references/test-authoring-guidance.md` (resolve the path relative to the repository root) as the positive rubric for test authoring. For any **computational** target (a computed numeric/algorithmic value — ratio, threshold, hash, conversion, round-trip, metric), assert against an **independent oracle** (reference library / published formula / hand-computed truth table — never the code's own output) on the **raw, pre-rounding** value with an explicit tolerance; add property / invariant tests (monotonicity, symmetry, idempotence, round-trip, containment); and cover adversarial / non-finite / out-of-range inputs by default (this default-adversarial coverage applies to **computational or behavioral** targets alike — M3, v8.4.0+ — any target whose function takes external / untrusted input, even though the oracle / raw-value / differential rules stay computational-only) — including at least one **parse-accepted-then-overflows** vector (a value the parser ACCEPTS that yields a non-finite / out-of-range intermediate, e.g. `oklch(0.5 1e400 30)` → Infinity chroma), not only parse-rejected `NaN` / `Infinity` tokens; and, when the function shares an input parser with sibling tools, ensure the input-validation guard lives in the SHARED boundary OR is replicated AND adversarially tested in EVERY sibling tool (see rule 4 of the guidance). A test that re-measures with the implementation's own rounded value is self-confirming and is rejected by tautological rule R4 (`skills/impl/references/tautological-assertion-rules.md`). At the `thorough` / `exhaustive` depth tier, for a standard-backed computational target author **two or more mutually-validated independent oracles** (at least one first-principles, no library) and trust a value only when they agree within tolerance — see `skills/impl/references/independent-oracle-harness.md`; ship a **committed, fixed-seed** property-fuzz loop (reproducible PRNG, tier-scaled case count) over the input distribution, not only fixed fixtures; and where a second INDEPENDENT ALGORITHM for the same contract exists, add an algorithm-vs-algorithm differential within tolerance (membership is necessary-not-sufficient). +Follow `skills/impl/references/test-authoring-guidance.md` (resolve the path relative to the repository root) as the positive rubric for test authoring. For any **computational** target (a computed numeric/algorithmic value — ratio, threshold, hash, conversion, round-trip, metric), assert against an **independent oracle** (reference library / published formula / hand-computed truth table — never the code's own output) on the **raw, pre-rounding** value with an explicit tolerance; add property / invariant tests (monotonicity, symmetry, idempotence, round-trip, containment); and cover adversarial / non-finite / out-of-range inputs by default (this default-adversarial coverage applies to **computational or behavioral** targets alike — M3, v8.4.0+ — any target whose function takes external / untrusted input, even though the oracle / raw-value / differential rules stay computational-only) — including at least one **parse-accepted-then-overflows** vector (an input the parser ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion — a `NaN` or `Infinity` sentinel surfacing downstream of an accepted value), not only parse-rejected `NaN` / `Infinity` tokens; and, when the function shares an input parser with sibling tools, ensure the input-validation guard lives in the SHARED boundary OR is replicated AND adversarially tested in EVERY sibling tool (see rule 4 of the guidance). A test that re-measures with the implementation's own rounded value is self-confirming and is rejected by tautological rule R4 (`skills/impl/references/tautological-assertion-rules.md`). At the `thorough` / `exhaustive` depth tier, for a standard-backed computational target author **two or more mutually-validated independent oracles** (at least one first-principles, no library) and trust a value only when they agree within tolerance — see `skills/impl/references/independent-oracle-harness.md`; ship a **committed, fixed-seed** property-fuzz loop (reproducible PRNG, tier-scaled case count) over the input distribution, not only fixed fixtures; and where a second INDEPENDENT ALGORITHM for the same contract exists, add an algorithm-vs-algorithm differential within tolerance (membership is necessary-not-sufficient). **Accept-set conformance retained corpus (always-on, `accept_set_conformance`, v8.5.0+).** Regardless of the `thorough` / `exhaustive` condition in the preceding sentence: when the ac-evaluator's accept-set conformance sweep reports an accept-set leak — an input the spec oracle rejects that a strict / canonical / lossless / limit boundary (or a same-input-class sibling) accepts, named in the evaluator Feedback — commit a **fixed rejection characterization test** to the PRODUCT tests for that leaking input class (the leaking input as a RED case; the validator fix is GREEN), so the regression is locked in the committed suite. This is the producer half of the two-surface retention: the evaluator's scratch sweep is ephemeral and never committed, so the durable test belongs here. Gated by `constraints.accept_set_conformance` (off → skip) but NOT by the depth tier — unlike the committed-seeded-fuzz clause above it is always-on at `standard` too. See `skills/impl/references/accept-set-conformance-harness.md`. ## Context Conservation Protocol diff --git a/agents/ticket-evaluator.md b/agents/ticket-evaluator.md index da9bf71..aaeae5b 100644 --- a/agents/ticket-evaluator.md +++ b/agents/ticket-evaluator.md @@ -56,7 +56,7 @@ Return format: - [x] Size Fit: description - [x] Capability Mapping: description (Gate 6 — applies only when at least one AC is runtime/visual per the canonical classifier; mark `n/a` when no AC triggers the classifier) - [x] Probe Completeness: description (Gate 6.5 — verifies every entry in the planner's `Available user skills:` / `Available MCP servers:` probe is classified Bound / Advisory / Skipped in the ticket; mark `n/a` when both probes report `(none)` or when the ticket pre-dates Gate 6.5) -- [x] Oracle Independence: description (Gate 7 — applies only when at least one AC is computational (a computed numeric/algorithmic value) per the canonical classifier; verifies each computational AC names an oracle independent of the implementation + a raw-value tolerance, OR declares the no-oracle fallback, AND (for a computational or behavioral AC on a function taking external input — broadened M3) requires adversarial / non-finite / out-of-range coverage — including at least one parse-accepted-then-overflows vector (e.g. `oklch(0.5 1e400 30)`), not only parse-rejected `NaN` / `Infinity` tokens — plus the sibling-guard requirement (a shared input-validation guard required across every sibling tool sharing the input boundary, not just one); and, for a standard-backed computational AC expected at the `thorough` / `exhaustive` evidence_floor, prefer two mutually-validated oracles with at least one first-principles (single oracle suffices at `standard`; absence of a second oracle is a Caveat, not a FAIL); where a second independent algorithm exists, prefer an algorithm-vs-algorithm differential (membership is necessary-not-sufficient; its absence is a Caveat, not a FAIL); mark `n/a` when no AC is computational, when the ticket sets `constraints.oracle_verification: off`, or when the ticket pre-dates Gate 7) +- [x] Oracle Independence: description (Gate 7 — applies only when at least one AC is computational (a computed numeric/algorithmic value) per the canonical classifier; verifies each computational AC names an oracle independent of the implementation + a raw-value tolerance, OR declares the no-oracle fallback, AND (for a computational or behavioral AC on a function taking external input — broadened M3) requires adversarial / non-finite / out-of-range coverage — including at least one parse-accepted-then-overflows vector (an input the parser ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion), not only parse-rejected `NaN` / `Infinity` tokens — plus the sibling-guard requirement (a shared input-validation guard required across every sibling tool sharing the input boundary — a set that is NOT confined to one ticket: when the spawn prompt forwards a cross-ticket `shared_input_boundary` membership for this ticket, the named siblings may live in separate tickets that each created a single unit — not just one; a member that DELEGATES to a shared parser still owes its own hostile / accept-set row or a justified `n/a` a reviewer can challenge, since delegation is NOT an automatic n/a and 'delegates to the shared parser' is not itself an acceptable n/a reason); and, for a standard-backed computational AC expected at the `thorough` / `exhaustive` evidence_floor, prefer two mutually-validated oracles with at least one first-principles (single oracle suffices at `standard`; absence of a second oracle is a Caveat, not a FAIL); where a second independent algorithm exists, prefer an algorithm-vs-algorithm differential (membership is necessary-not-sufficient; its absence is a Caveat, not a FAIL); mark `n/a` when no AC is computational, when the ticket sets `constraints.oracle_verification: off`, or when the ticket pre-dates Gate 7) - [x] Peer-Set Uniformity: description (Gate 10 — applies only when the ticket's `### Scope` creates a `>=2`-peer set (a family of analogous sibling units in one category — peer tools / endpoints / subcommands / exported functions sharing an output surface) per the canonical classifier; verifies the ticket carries `>=1` UNIFIED-convention AC over the set (single error convention / single success-envelope shape / single vocabulary per concept / single wrapper for repeated boilerplate, mechanically grep/AST-verifiable across every peer) OR a one-line `n/a` justification under `#### Peer-Set Uniformity (Gate 10)` — a peer set with neither is a FAIL, a justified `n/a` is a PASS. Gate 10 ENUMERATES coverage presence for the peer set; it does NOT re-grade which convention was unified (the diff-time L-UNIFORMITY lens grades the code) and MUST NOT FAIL merely because the evaluator would have chosen a different convention. Disjoint from Gate 7's sibling-INPUT-guard and from Gate 9's per-boundary matrix. Mark `n/a` when the Scope creates `<2` peers, when the ticket sets `constraints.peer_uniformity: off`, or when the ticket pre-dates Gate 10) - [x] Failure-Class Coverage: description (Gate 9 — applies only when the ticket's `### Scope` touches >=1 external boundary (public / exported function, CLI subcommand, endpoint, exported API symbol, file-format / wire-format, parser, or a `>=2`-peer sibling set) per the canonical classifier; verifies the `#### Failure-Class Coverage (Gate 9)` matrix carries, for EACH such boundary, four rows (R1 FULL-DOMAIN INVARIANT, R2 HOSTILE + BOUNDED TERMINATION + RESOURCE-CAP, R3 DESCRIPTION-MATCHES-BEHAVIOR, R4 DOC/INTERFACE TRUTHFULNESS) each resolved to >=1 AC OR a one-line `n/a` justification — a blank row is a FAIL, a justified `n/a` is a PASS. Gate 9 ENUMERATES coverage presence per boundary; it does NOT re-grade AC adequacy (Gates 1-8 do that) and MUST NOT FAIL merely because the evaluator would have written a different AC. Mark `n/a` when the ticket touches no external boundary (internal-helper-only — routine-ticket flood prevention), when the ticket sets `constraints.failure_class_coverage: off`, or when the ticket pre-dates Gate 9) - [x] Independent Evidence: description (Gate 8 — applies only when at least one AC is behavioral (PASS/FAIL hinges on observable runtime behaviour) per the canonical classifier; verifies each behavioral AC names at least one evidence channel independent of the implementation — EC-ORACLE / EC-DIFFERENTIAL / EC-PROPERTY / EC-RUNTIME per `skills/impl/references/evidence-channels.md` — OR is rewritten as a structural AC (EC-STATIC). A computational AC satisfies Gate 8 via its Gate 7 oracle. The AC's natural channel counts (a black-box CLI / API assertion is EC-RUNTIME) — do NOT FAIL a behavioral AC whose natural evidence is already independent. For a computational / algorithmic AC expected at the `thorough` / `exhaustive` evidence_floor, prefer a committed, fixed-seed property-fuzz loop over the input distribution (EC-PROPERTY); its absence is a Caveat, not a FAIL. Mark `n/a` when every AC is purely structural, when the ticket sets `constraints.independent_evidence: off`, or when the ticket pre-dates Gate 8) diff --git a/design-oracles/aasc-accept-set/README.md b/design-oracles/aasc-accept-set/README.md new file mode 100644 index 0000000..574563d --- /dev/null +++ b/design-oracles/aasc-accept-set/README.md @@ -0,0 +1,48 @@ +# AASC accept-set-conformance — design-oracle (proof prototype) + +This directory is a **committed design-oracle**: a runnable proof that the +**Advertised-Accept-Set Conformance (AASC)** mechanism is language-agnostic, +constructive, and false-positive-free. It satisfies the charter `§F-1` evidence +item ("commit the prototype as a repo design-oracle"). It is a **proof artifact, +not normative plugin content** — the AASC mechanism itself is NOT yet wired into +the harness; its implementation is gated on the charter `§F-4` discriminating-subject +live dogfood. Nothing here is run by the test suites or by CI. + +## What it proves + +The probes pair an **independent, hand-coded spec oracle** (accept/reject derived +from the advertised grammar, never importing the implementation) with a +**property-generated complement corpus**: the generator enumerates the Unicode +decimal-digit codepoints that are NOT ASCII (the `\p{Nd}` / Unicode-category +complement of `U+0030..U+0039`), across the BMP **and the astral planes**, **naming +no script in code**. It then runs each implementation **black-box** and diffs every +verdict against the oracle, with accepted-domain idempotence. + +The boundary under test (a strict unsigned decimal octet `[0,255]`) is **illustrative +only** — illustrative on both the domain axis and the language axis; the mechanism is +the point, not the example. + +## Results (reproducible) + +- `node probe.js` — the buggy "looks-numeric-and-in-range" implementation is caught + (its leaks are structural / non-canonical: leading-zero, whitespace, hex, scientific + notation, empty); `astral among catches: 0` is **correct** here, because the JS + native numeric parser is ASCII-only and so leaks no Unicode alphabet. The correct + implementation has **zero** oracle-disagreements (no false-positive storm). +- `python3 probe.py` — the buggy `str.isdigit()`-then-`int()` implementation leaks the + **full Unicode-Nd alphabet including ~390 astral codepoints** (the exact class a prior + dogfood shipped), **caught by construction** with the generator naming no script. The + correct implementation has **zero** oracle-disagreements. + +The same methodology (independent oracle × property-generated complement corpus) +catches each language's *actual* leak class — structural for the ASCII-native parser, +input-alphabet for the Unicode-permissive parser — with one oracle and no false +positives. That is the language-agnostic, constructive, false-positive-free property +AASC rests on. + +## Run + +``` +node design-oracles/aasc-accept-set/probe.js +python3 design-oracles/aasc-accept-set/probe.py +``` diff --git a/design-oracles/aasc-accept-set/probe.js b/design-oracles/aasc-accept-set/probe.js new file mode 100644 index 0000000..02a9581 --- /dev/null +++ b/design-oracles/aasc-accept-set/probe.js @@ -0,0 +1,73 @@ +'use strict'; +// AASC confidence probe (JS). Boundary: strict unsigned decimal octet [0,255]. +// Advertised accept-set: ASCII digits only, canonical (no leading zero), +// no sign / whitespace / radix prefix, value 0..255. +// The oracle is SPEC-DERIVED and INDEPENDENT of both impls: it does a hand +// char-code check and never calls Number()/parseInt — so it cannot share an +// impl's parsing convention. + +// ---- Independent spec oracle (no library, no impl reuse) ---- +function oracleAccepts(s) { + if (typeof s !== 'string' || s.length === 0) return false; + for (const ch of s) { // for..of iterates by code point (astral-safe) + const cp = ch.codePointAt(0); + if (cp < 0x30 || cp > 0x39) return false; // ASCII digits only + } + if (s.length > 1 && s[0] === '0') return false; // canonical: no leading zero + let v = 0; + for (const ch of s) { v = v * 10 + (ch.codePointAt(0) - 0x30); if (v > 255) return false; } + return true; +} + +// ---- Impls under test ---- +function implCorrect(s) { + return /^(0|[1-9][0-9]{0,2})$/.test(s) && Number(s) <= 255; +} +function implBuggy(s) { // "looks numeric and in range" shortcut + const n = Number(s); + return Number.isInteger(n) && n >= 0 && n <= 255; +} + +// ---- GENERATIVE corpus (grammar/property-driven; NAMES NO SCRIPT) ---- +const corpus = []; +const add = (s) => corpus.push(s); +['0','1','9','10','42','99','100','199','255'].forEach(add); // ascii accepts +['256','300','999','00','01','0255','+1','-1','1.0','1e2','0x1f','0b1','1_0','',' 12','12 ',' ','\t9'].forEach(add); // ascii rejects +// Unicode decimal-digit COMPLEMENT sweep: every codepoint that is \p{Nd} +// but NOT ASCII 0-9. Enumerated by PROPERTY — recalls no script name. +let ndBmp = 0, ndAstral = 0; +for (let cp = 0; cp <= 0x10FFFF; cp++) { + if (cp >= 0xD800 && cp <= 0xDFFF) continue; // lone surrogates + if (cp >= 0x30 && cp <= 0x39) continue; // exclude ASCII + const ch = String.fromCodePoint(cp); + if (/\p{Nd}/u.test(ch)) { add(ch); if (cp > 0xFFFF) ndAstral++; else ndBmp++; } +} +['²','Ⅰ','½','〇'].forEach(add); // non-Nd numeric lookalikes (must reject) + +// ---- Run + diff vs oracle ---- +function run(impl) { + const fa = [], fr = [], threw = []; + for (const s of corpus) { + const want = oracleAccepts(s); + let got; + try { got = impl(s); } catch (_) { threw.push(s); continue; } + if (got && !want) fa.push(s); + if (!got && want) fr.push(s); + } + return { fa, fr, threw }; +} +const hex = (s) => [...s].map(c => 'U+' + c.codePointAt(0).toString(16).toUpperCase().padStart(4, '0')).join(''); +const sample = (a, n) => a.slice(0, n).map(s => JSON.stringify(s) + '(' + hex(s) + ')').join(', '); + +const rb = run(implBuggy), rc = run(implCorrect); +console.log('== JS probe :: boundary = strict unsigned decimal octet [0,255] =='); +console.log('corpus size:', corpus.length, '| nd-nonascii BMP:', ndBmp, 'astral:', ndAstral); +console.log('-- BUGGY impl (Number()+range shortcut) --'); +console.log(' false-accepts (leak caught by oracle):', rb.fa.length); +console.log(' sample:', sample(rb.fa, 14)); +const buggyAstral = rb.fa.filter(s => [...s].some(c => c.codePointAt(0) > 0xFFFF)); +console.log(' astral among catches:', buggyAstral.length); +console.log('-- CORRECT impl (regex+value) --'); +const dis = rc.fa.length + rc.fr.length + rc.threw.length; +console.log(' disagreements with oracle:', dis, '->', dis === 0 ? 'false-positive-storm ABSENT (0)' : 'STORM PRESENT'); +console.log('VERDICT JS: buggy caught =', rb.fa.length > 0, '| correct clean =', dis === 0); diff --git a/design-oracles/aasc-accept-set/probe.py b/design-oracles/aasc-accept-set/probe.py new file mode 100644 index 0000000..2e8c128 --- /dev/null +++ b/design-oracles/aasc-accept-set/probe.py @@ -0,0 +1,82 @@ +# AASC confidence probe (Python). Same boundary: strict unsigned decimal octet [0,255]. +# Oracle is SPEC-DERIVED and INDEPENDENT of both impls: hand char-code check, +# never calls int()/str.isdigit(). +import unicodedata, json + +def oracle_accepts(s): + if not isinstance(s, str) or len(s) == 0: + return False + for ch in s: + if ord(ch) < 0x30 or ord(ch) > 0x39: # ASCII digits only + return False + if len(s) > 1 and s[0] == '0': # canonical: no leading zero + return False + v = 0 + for ch in s: + v = v * 10 + (ord(ch) - 0x30) + if v > 255: + return False + return True + +import re +_pat = re.compile(r'(0|[1-9][0-9]{0,2})') # ASCII literals -> ASCII-only by construction +def impl_correct(s): + return _pat.fullmatch(s) is not None and int(s) <= 255 + +def impl_buggy(s): # "looks like digits and in range" shortcut + return s.isdigit() and 0 <= int(s) <= 255 + +# GENERATIVE corpus (property-driven; names no script) +corpus = [] +for s in ['0','1','9','10','42','99','100','199','255']: + corpus.append(s) +for s in ['256','300','999','00','01','0255','+1','-1','1.0','1e2','0x1f','',' ',' 12','12 ','\t9']: + corpus.append(s) +nd_bmp = nd_astral = 0 +for cp in range(0x110000): + if 0x30 <= cp <= 0x39 or 0xD800 <= cp <= 0xDFFF: + continue + ch = chr(cp) + if unicodedata.category(ch) == 'Nd': # Unicode decimal digit, non-ASCII + corpus.append(ch) + if cp > 0xFFFF: + nd_astral += 1 + else: + nd_bmp += 1 +for s in ['²', 'Ⅰ', '½', '〇']: # non-Nd numeric lookalikes + corpus.append(s) + +def run(impl): + fa, fr, threw = [], [], [] + for s in corpus: + want = oracle_accepts(s) + try: + got = impl(s) + except Exception: + threw.append(s) + continue + if got and not want: + fa.append(s) + if (not got) and want: + fr.append(s) + return fa, fr, threw + +def hexs(s): + return ''.join('U+%04X' % ord(c) for c in s) +def sample(a, n): + return ', '.join(json.dumps(x) + '(' + hexs(x) + ')' for x in a[:n]) + +fb, frb, tb = run(impl_buggy) +fc, frc, tc = run(impl_correct) +print('== PY probe :: boundary = strict unsigned decimal octet [0,255] ==') +print('corpus size:', len(corpus), '| nd-nonascii BMP:', nd_bmp, 'astral:', nd_astral) +print('-- BUGGY impl (str.isdigit()+int()+range) --') +print(' false-accepts (leak caught by oracle):', len(fb)) +print(' sample:', sample(fb, 14)) +astral_caught = [s for s in fb if any(ord(c) > 0xFFFF for c in s)] +print(' ASTRAL among catches:', len(astral_caught), ('(e.g. ' + sample(astral_caught, 3) + ')') if astral_caught else '') +print(' threw (robustness defect, e.g. int() on digit-ish non-Nd):', len(tb), ('(e.g. ' + sample(tb, 4) + ')') if tb else '') +print('-- CORRECT impl (ascii regex + value) --') +dis = len(fc) + len(frc) + len(tc) +print(' disagreements with oracle:', dis, '->', 'false-positive-storm ABSENT (0)' if dis == 0 else 'STORM PRESENT') +print('VERDICT PY: buggy caught =', len(fb) > 0, '| ASTRAL leak caught =', len(astral_caught) > 0, '| correct clean =', dis == 0) diff --git a/design-oracles/aasc-keyfaith/README.md b/design-oracles/aasc-keyfaith/README.md new file mode 100644 index 0000000..3c5852c --- /dev/null +++ b/design-oracles/aasc-keyfaith/README.md @@ -0,0 +1,69 @@ +# AASC MR-KEYFAITH (class b) — design-oracle (proof prototype) + +This directory is a **committed design-oracle**: a runnable proof that the +**class (b) MR-KEYFAITH** mechanism — a reflection-derived dangerous-key corpus +that names no key literal, paired with an independent round-trip-faithfulness +oracle — is language-agnostic, constructive, and false-positive-free. It is the +keyed-structure (K-axis) sibling of `../aasc-accept-set/` (the MR-ALPHABET / +class (c) proof). It is a **proof artifact, not normative plugin content** — the +mechanism's prose lives in `skills/impl/references/accept-set-conformance-harness.md` +(§ The worked MR-KEYFAITH (b) shape) and the `ac-evaluator` / `ac-evaluator-hi` +L-ROBUSTNESS lens. Nothing here is run by the test suites or by CI. + +## What it proves + +The probes pair an **independent round-trip-faithfulness oracle** with a +**reflection-derived key corpus**: + +- The oracle computes the expectation **from the INPUT PAIRS by last-write-wins** + and **never reads it back out of the builder** — so a builder that drops or + overwrites a key cannot "agree with itself" (the circular-oracle trap). It then + checks every input key reads back ITS value through the public getter, and that + `build(serialize(x))` preserves the same observable mapping (the round-trip leg). +- The key generator **DERIVES** the dangerous accessor / reserved / **private / + internal slot** names **BY REFLECTION** over the live host structures (JS: + `Object.getOwnPropertySymbols` + climbing `getPrototypeOf` collecting + `Object.getOwnPropertyNames`; Python: `dir(type(target))` + walking + `type(target).__mro__` and collecting each base's `vars(base)` keys), **naming + no key literal in the generator** — exactly as the MR-ALPHABET probe selects + digits by the Unicode decimal-digit PROPERTY and names no script. Plus the + generic structural hostiles (empty key, duplicate of an existing key, a + normalization-collision partner derived from an existing key). + +Each probe runs a CORRECT builder (a null-prototype container / a dict-backed +store — the key namespace cannot collide with the structure's reserved slots) and +a BUGGY builder (a plain `{}` with `obj[k]=v`, plus a nested deep-assign variant / +a class doing `setattr` per pair into attribute space) **black-box** over the +reflected corpus, and diffs every verdict against the oracle. + +The boundary under test (an in-memory keyed store rebuilt from `(key, value)` +pairs) is **illustrative only** — illustrative on both the domain axis and the +language axis; the mechanism (reflection-derived corpus × independent round-trip +oracle) is the point, not the example. + +## Results (reproducible) + +- `node probe.js` — the buggy plain-`{}` builder is caught: an input key that + shadows a prototype slot is not stored as a faithful own-property mapping (the + flat and deep variants both surface the violation). The correct + `Object.create(null)` builder (flat and deep) has **zero** oracle-disagreements + (no false-positive storm). +- `python3 probe.py` — the buggy `setattr`-into-attribute-space builder leaks its + **private / internal slot** collisions (the input key collides with the + instance's own internal storage slots, surfaced by the reflection-derived + corpus that includes those names), **caught by construction** with the generator + naming no key. The correct dict-backed store has **zero** oracle-disagreements. + +The same methodology (independent round-trip-faithfulness oracle × reflection- +derived corpus naming no key) catches each language's *actual* keyed-structure +leak class — prototype-slot shadowing for the plain-object builder, internal-slot +collision for the attribute-space builder — with one oracle and no false +positives. That is the language-agnostic, constructive, false-positive-free +property the class (b) MR-KEYFAITH mechanism rests on. + +## Run + +``` +node design-oracles/aasc-keyfaith/probe.js +python3 design-oracles/aasc-keyfaith/probe.py +``` diff --git a/design-oracles/aasc-keyfaith/probe.js b/design-oracles/aasc-keyfaith/probe.js new file mode 100644 index 0000000..e516693 --- /dev/null +++ b/design-oracles/aasc-keyfaith/probe.js @@ -0,0 +1,221 @@ +'use strict'; +// AASC MR-KEYFAITH confidence probe (JS). Class (b): a KEYED structure rebuilt +// from untrusted (key, value) input pairs. Advertised contract: round-trip +// faithfulness — every key the inputs asked for returns ITS value unchanged +// (last-write-wins for a repeated key), no silent drop, no host-metadata bleed, +// and build(serialize(x)) preserves the same observable mapping. +// +// The oracle is SPEC-DERIVED and INDEPENDENT of every builder: it computes the +// expectation from the INPUT PAIRS by last-write-wins and NEVER reads it back out +// of a builder (the circular-oracle trap). The key generator names NO key literal: +// it DERIVES the dangerous accessor / reserved / private names BY REFLECTION over +// the live host structures, exactly as the (c) probe selects digits by the Unicode +// decimal-digit PROPERTY and names no script. +// +// Two self-consistent comparisons share one corpus of reflected keys: +// FLAT — obj[k]=v, oracle = flat last-write-wins map. +// DEEP — a nested "k.split('.')" deep-assign variant, oracle = deep +// last-write-wins map. Each oracle matches its builders' key model, so a +// CORRECT builder is a clean zero in both. + +const MISSING = Symbol('MISSING'); + +// ===================== FLAT comparison ===================== + +// Independent flat oracle: expectation from the INPUT PAIRS by last-write-wins. +function oracleFlat(pairs, build, get, serialize) { + const expected = new Map(); + for (const [k, v] of pairs) expected.set(k, v); // later pair for a repeated key wins + const built = build(pairs); + for (const [k, want] of expected) { + if (get(built, k) !== want) return false; // silent drop or wrong-value overwrite + } + const again = build(serialize(built)); // round-trip: serialize then build + for (const [k, want] of expected) { + if (get(again, k) !== want) return false; + } + return true; +} + +// CORRECT: a null-prototype container — no inherited accessor / reserved slot can +// be shadowed or mutated; every own key round-trips. +function buildFlatCorrect(pairs) { + const o = Object.create(null); + for (const [k, v] of pairs) o[String(k)] = v; // last-write-wins + return o; +} +// BUGGY: a plain {} — an accessor / reserved name (read off the live host by +// reflection) can shadow a prototype slot, return the inherited value, or mutate +// host-structure metadata instead of storing the pair. +function buildFlatBuggy(pairs) { + const o = {}; // inherits the host object prototype + for (const [k, v] of pairs) o[String(k)] = v; + return o; +} +function getFlat(o, k) { + return Object.prototype.hasOwnProperty.call(o, k) ? o[k] : MISSING; // own-property read only +} +function serializeFlat(o) { + return Object.keys(o).map(k => [k, o[k]]); // own enumerable wire form +} + +// ===================== DEEP comparison ===================== + +// Independent deep oracle: nested last-write-wins map keyed by "k.split('.')". +function setDeep(map, segs, v) { + let cur = map; + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + if (!(cur.get(s) instanceof Map)) cur.set(s, new Map()); + cur = cur.get(s); + } + cur.set(segs[segs.length - 1], v); +} +function getDeep(map, segs) { + let cur = map; + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + if (!(cur instanceof Map) || !cur.has(s)) return MISSING; + cur = cur.get(s); + } + return (cur instanceof Map && cur.has(segs[segs.length - 1])) ? cur.get(segs[segs.length - 1]) : MISSING; +} +function oracleDeep(pairs, build, getPath, serialize) { + const expected = new Map(); + for (const [k, v] of pairs) setDeep(expected, String(k).split('.'), v); + const built = build(pairs); + const want = []; + (function walk(m, prefix) { + for (const [k, v] of m) { + if (v instanceof Map) walk(v, prefix.concat(k)); + else want.push([prefix.concat(k), v]); + } + })(expected, []); + for (const [segs, v] of want) { + if (getPath(built, segs) !== v) return false; + } + const again = build(serialize(built)); + for (const [segs, v] of want) { + if (getPath(again, segs) !== v) return false; + } + return true; +} + +// CORRECT deep: null-proto at every level — no reserved slot collision. +function buildDeepCorrect(pairs) { + const root = Object.create(null); + for (const [k, v] of pairs) { + const segs = String(k).split('.'); + let cur = root; + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + if (typeof cur[s] !== 'object' || cur[s] === null) cur[s] = Object.create(null); + cur = cur[s]; + } + cur[segs[segs.length - 1]] = v; + } + return root; +} +// BUGGY deep: deep-assign onto plain {} — a reserved segment pollutes / shadows. +function buildDeepBuggy(pairs) { + const root = {}; + for (const [k, v] of pairs) { + const segs = String(k).split('.'); + let cur = root; + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + if (typeof cur[s] !== 'object' || cur[s] === null) cur[s] = {}; + cur = cur[s]; + } + cur[segs[segs.length - 1]] = v; + } + return root; +} +function getDeepPath(root, segs) { + let cur = root; + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + if (typeof cur !== 'object' || cur === null || !Object.prototype.hasOwnProperty.call(cur, s)) return MISSING; + cur = cur[s]; + } + const last = segs[segs.length - 1]; + return (typeof cur === 'object' && cur !== null && Object.prototype.hasOwnProperty.call(cur, last)) ? cur[last] : MISSING; +} +function serializeDeep(root) { + const out = []; + (function walk(o, prefix) { + for (const k of Object.keys(o)) { + const v = o[k]; + if (v && typeof v === 'object' && !Array.isArray(v)) walk(v, prefix.concat(k)); + else out.push([prefix.concat(k).join('.'), v]); + } + })(root, []); + return out; +} + +// ===================== REFLECTION-DERIVED key generator (NAMES NO KEY LITERAL) ===================== +// Climb the prototype chain of live host structures, collecting every own property +// name + symbol at every level. The accessor / reserved / private slot names are +// whatever the runtime exposes — no denylist, no hard-coded key. +function reflectHostileKeys() { + const keys = new Set(); + for (const target of [{}, new Map(), []]) { + for (let p = target; p; p = Object.getPrototypeOf(p)) { + for (const n of Object.getOwnPropertyNames(p)) keys.add(n); + for (const s of Object.getOwnPropertySymbols(p)) keys.add(s); + } + } + return keys; +} +function makeKeyCorpus(existingKeys) { + const hostile = reflectHostileKeys(); // reflection-derived accessor / reserved names + const generic = new Set(); + generic.add(''); // empty key (generic structural hostile) + if (existingKeys.length) generic.add(existingKeys[0]); // duplicate of an existing key + for (const k of existingKeys) { // normalized-collision partner, no literal pair + if (typeof k === 'string' && k !== k.normalize('NFC')) { generic.add(k.normalize('NFC')); break; } + } + return [...hostile, ...generic].filter(k => typeof k === 'string'); // symbols have no string wire form +} + +// Synthetic-value pairs; the candidate key is the only variable (+ a repeat for +// last-write-wins). Benign anchors must always round-trip. +const seedKeys = ['alpha', 'beta']; +const hostileKeys = makeKeyCorpus(seedKeys); +const flatPairs = (k) => [['alpha', 1], ['beta', 2], [k, 'V:' + k], [k, 'V2:' + k]]; +const deepPairs = (k) => [['x.alpha', 1], ['x.beta', 2], [k, 'V:' + k], [k, 'V2:' + k]]; + +function run(build, get, serialize, oracle, pairsFor) { + const violations = []; + for (const k of hostileKeys) { + let ok; + try { ok = oracle(pairsFor(k), build, get, serialize); } + catch (_) { violations.push({ k, reason: 'threw' }); continue; } + if (!ok) violations.push({ k, reason: 'unfaithful' }); + } + return violations; +} + +const flatBuggyV = run(buildFlatBuggy, getFlat, serializeFlat, oracleFlat, flatPairs); +const flatCorrectV = run(buildFlatCorrect, getFlat, serializeFlat, oracleFlat, flatPairs); +const deepBuggyV = run(buildDeepBuggy, getDeepPath, serializeDeep, oracleDeep, deepPairs); +const deepCorrectV = run(buildDeepCorrect, getDeepPath, serializeDeep, oracleDeep, deepPairs); + +const hex = (k) => [...k].map(c => 'U+' + c.codePointAt(0).toString(16).toUpperCase().padStart(4, '0')).join(''); +const sample = (a, n) => a.slice(0, n).map(x => JSON.stringify(x.k) + '(' + hex(x.k) + ',' + x.reason + ')').join(', '); + +console.log('== JS probe :: class (b) MR-KEYFAITH — keyed structure round-trip faithfulness =='); +console.log('reflection-derived hostile keys (string-serializable):', hostileKeys.length, '| names no key literal in generator'); +console.log('-- BUGGY builder (plain {} + obj[k]=v, flat) --'); +console.log(' faithfulness violations (caught by oracle):', flatBuggyV.length); +console.log(' sample:', sample(flatBuggyV, 8)); +console.log('-- BUGGY builder (deep-assign onto plain {}) --'); +console.log(' faithfulness violations (caught by oracle):', deepBuggyV.length); +console.log(' sample:', sample(deepBuggyV, 8)); +console.log('-- CORRECT builder (Object.create(null), flat) --'); +console.log(' oracle disagreements:', flatCorrectV.length, '->', flatCorrectV.length === 0 ? 'false-positive-storm ABSENT (0)' : 'STORM PRESENT'); +console.log('-- CORRECT builder (deep null-proto) --'); +console.log(' oracle disagreements:', deepCorrectV.length, '->', deepCorrectV.length === 0 ? 'false-positive-storm ABSENT (0)' : 'STORM PRESENT'); +const buggyCaught = flatBuggyV.length > 0 && deepBuggyV.length > 0; +const correctClean = flatCorrectV.length + deepCorrectV.length; +console.log('VERDICT JS: buggy caught =', buggyCaught, '| correct clean =', correctClean); diff --git a/design-oracles/aasc-keyfaith/probe.py b/design-oracles/aasc-keyfaith/probe.py new file mode 100644 index 0000000..cd37682 --- /dev/null +++ b/design-oracles/aasc-keyfaith/probe.py @@ -0,0 +1,158 @@ +# AASC MR-KEYFAITH confidence probe (Python). Class (b): a KEYED structure rebuilt +# from untrusted (key, value) input pairs. Advertised contract: round-trip +# faithfulness — every key the inputs asked for returns ITS value unchanged +# (last-write-wins for a repeated key), no silent drop, no internal-slot bleed, +# and build(serialize(x)) preserves the same observable mapping. +# +# The oracle is SPEC-DERIVED and INDEPENDENT of both builders: it computes the +# expectation from the INPUT PAIRS by last-write-wins and NEVER reads it back out +# of a builder (the circular-oracle trap). The key generator names NO key literal: +# it DERIVES the dangerous reserved / accessor / private-slot names BY REFLECTION +# over the live type (dir + the __mro__ vars walk), exactly as the (c) probe +# selects digits by the Unicode decimal-digit PROPERTY and names no script. +import json + +_MISSING = object() + + +# ---- Independent round-trip-faithfulness oracle (no builder reuse) ---- +# `expected` is derived ONLY from the input pairs (last-write-wins). It is never +# read out of a builder, so a builder that drops a key cannot "agree with itself". +def oracle_faithful(pairs, build, get, serialize): + expected = {} + for k, v in pairs: + expected[k] = v # later pair for a repeated key wins + built = build(pairs) + for k, want in expected.items(): + if get(built, k) != want: # silent drop or wrong-value overwrite + return False + again = build(serialize(built)) # round-trip: serialize then build + for k, want in expected.items(): + if get(again, k) != want: + return False + return True + + +# ---- Builders under test ---- +# BUGGY: a class that does setattr per pair into ATTRIBUTE space — an input key +# that collides with the class's own reserved / accessor / private slot name (read +# off the live type by reflection) overwrites that slot or is shadowed by it, +# instead of being stored as data. Reads come back through getattr. +class _AttrStore: + def __init__(self, pairs): + for k, v in pairs: + setattr(self, str(k), v) # last-write-wins, but pollutes attribute space + + +def build_buggy(pairs): + return _AttrStore(pairs) + + +def get_buggy(store, k): + return getattr(store, k, _MISSING) + + +def serialize_buggy(store): + # wire form = the instance's own __dict__ items (data attributes only). + return list(vars(store).items()) + + +# CORRECT: a dict-backed store — keys live in a data namespace that cannot collide +# with the type's reserved / accessor / private slots; every key round-trips. +class _DictStore: + def __init__(self, pairs): + self._d = {} + for k, v in pairs: + self._d[str(k)] = v # last-write-wins, isolated namespace + + +def build_correct(pairs): + return _DictStore(pairs) + + +def get_correct(store, k): + return store._d.get(k, _MISSING) + + +def serialize_correct(store): + return list(store._d.items()) + + +# ---- REFLECTION-DERIVED key generator (NAMES NO KEY LITERAL) ---- +# Read the reserved / accessor / private-slot names off the live types by +# reflection — dir(type(target)) plus walking type(target).__mro__ and collecting +# each base's vars(base) keys. This INCLUDES private / internal slot names +# (leading-underscore / name-mangled / dunder); no name is hard-coded. +def reflect_hostile_keys(): + keys = set() + for target in ({}, [], _AttrStore([]), _DictStore([])): # live host structures + t = type(target) + keys.update(dir(t)) # the type's own method + accessor + dunder names + for base in t.__mro__: # walk the resolution order to the root + keys.update(vars(base).keys()) # each ancestor's own attribute names (incl private slots) + return keys + + +def make_key_corpus(existing_keys): + hostile = reflect_hostile_keys() # reflection-derived reserved / accessor / private names + generic = set() + generic.add('') # empty key (generic structural hostile) + if existing_keys: + generic.add(next(iter(existing_keys))) # duplicate of an existing key + # normalized-collision partner: derived from an existing key, no literal pair. + for k in existing_keys: + if isinstance(k, str) and k != k.casefold(): + generic.add(k.casefold()) + break + return hostile | generic + + +seed_keys = ['alpha', 'Beta'] # benign anchors that must always round-trip +hostile_keys = sorted(k for k in make_key_corpus(seed_keys) if isinstance(k, str)) + + +def pairs_for(k): + # benign anchors + candidate key (synthetic value) + repeated key (last-write-wins). + return [('alpha', 1), ('Beta', 2), (k, 'V:' + k), (k, 'V2:' + k)] + + +def run(build, get, serialize): + violations = [] + for k in hostile_keys: + pairs = pairs_for(k) + try: + ok = oracle_faithful(pairs, build, get, serialize) + except Exception as e: + violations.append((k, 'threw:' + type(e).__name__)) + continue + if not ok: + violations.append((k, 'unfaithful')) + return violations + + +def hexs(s): + return ''.join('U+%04X' % ord(c) for c in s) + + +def sample(a, n): + return ', '.join(json.dumps(k) + '(' + hexs(k) + ',' + r + ')' for k, r in a[:n]) + + +buggy_v = run(build_buggy, get_buggy, serialize_buggy) +correct_v = run(build_correct, get_correct, serialize_correct) +priv_caught = [(k, r) for k, r in buggy_v if k.startswith('_')] + +print('== PY probe :: class (b) MR-KEYFAITH — keyed structure round-trip faithfulness ==') +print('reflection-derived hostile keys:', len(hostile_keys), '| names no key literal in generator') +print('-- BUGGY builder (setattr into attribute space) --') +print(' faithfulness violations (caught by oracle):', len(buggy_v)) +print(' sample:', sample(buggy_v, 10)) +print(' private/internal-slot collisions among catches:', len(priv_caught), + ('(e.g. ' + sample(priv_caught, 4) + ')') if priv_caught else '') +print('-- CORRECT builder (dict-backed store) --') +dis = len(correct_v) +print(' oracle disagreements:', dis, '->', + 'false-positive-storm ABSENT (0)' if dis == 0 else 'STORM PRESENT') +print('VERDICT PY: buggy caught =', len(buggy_v) > 0, + '| private-slot leak caught =', len(priv_caught) > 0, + '| correct clean =', dis == 0) diff --git a/docs/brief-chain-flow-dark.png b/docs/brief-chain-flow-dark.png new file mode 100644 index 0000000..1c8edd6 Binary files /dev/null and b/docs/brief-chain-flow-dark.png differ diff --git a/docs/brief-chain-flow.png b/docs/brief-chain-flow.png new file mode 100644 index 0000000..7db0fbd Binary files /dev/null and b/docs/brief-chain-flow.png differ diff --git a/docs/brief-chain-overview-dark.png b/docs/brief-chain-overview-dark.png new file mode 100644 index 0000000..e4e376d Binary files /dev/null and b/docs/brief-chain-overview-dark.png differ diff --git a/docs/brief-chain-overview.png b/docs/brief-chain-overview.png new file mode 100644 index 0000000..e54961f Binary files /dev/null and b/docs/brief-chain-overview.png differ diff --git a/docs/gen-brief-chain-flow.py b/docs/gen-brief-chain-flow.py new file mode 100644 index 0000000..c72666f --- /dev/null +++ b/docs/gen-brief-chain-flow.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""Generate the detailed `/brief chain=on` execution flow (light + dark). + +Renders, with Pillow only (no external diagram toolchain), an annotated flowchart +of the default `/brief chain=on` path: what each phase does, which subagents it +spawns, what it writes, what is handed off *between* phases, which lifecycle hooks +fire, where the two nested loops are, and which harness mechanism governs each +step. Collapsed under a
in README.md. Emits two files for a +dark-mode swap: + + docs/brief-chain-flow.png (light) + docs/brief-chain-flow-dark.png (dark) + +Re-generate from anywhere: python3 docs/gen-brief-chain-flow.py +Glyph rule: Arial lacks ↻ ↺ ★ ⇄ ≤ (they render as tofu boxes), so we use only +ASCII plus the safe arrows → · — × and draw loop/marker glyphs as shapes. +""" + +import os +from PIL import Image, ImageDraw, ImageFont + +# ---------------------------------------------------------------- canvas / scale +S = 2 +W, H = 1200, 2000 +HERE = os.path.dirname(os.path.abspath(__file__)) + +# ------------------------------------------------------------------------- fonts +_FONT_CANDIDATES = { + False: ["/System/Library/Fonts/Supplemental/Arial.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", "/Library/Fonts/Arial.ttf"], + True: ["/System/Library/Fonts/Supplemental/Arial Bold.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", "/Library/Fonts/Arial Bold.ttf"], +} +_font_cache = {} + + +def font(size, bold=False): + key = (round(size, 1), bold) + if key in _font_cache: + return _font_cache[key] + f = None + for path in _FONT_CANDIDATES[bold]: + if os.path.exists(path): + try: + f = ImageFont.truetype(path, int(round(size * S))) + break + except Exception: + pass + if f is None: + f = ImageFont.load_default() + _font_cache[key] = f + return f + + +def P(v): + return int(round(v * S)) + + +# ------------------------------------------------------------------------ themes +# Every name used by the drawing body is supplied per theme via globals().update, +# so the same layout renders in light or dark. Names that previously were inline +# tuples (diamond / separator / hook-name / harness-desc) are themed too, since a +# hard-coded dark brown would vanish on a dark background. +LIGHT = { + "BG": (255, 255, 255), "LABELBG": (255, 255, 255), + "INK": (17, 24, 39), "GRAY": (107, 114, 128), "LINE": (71, 85, 105), "SLATE": (71, 85, 105), + "SKILL_FILL": (219, 234, 254), "SKILL_BD": (37, 99, 235), "SKILL_TX": (30, 58, 138), + "AGENT_FILL": (220, 252, 231), "AGENT_BD": (22, 163, 74), "AGENT_TX": (20, 83, 45), + "HOOK_FILL": (255, 250, 235), "HOOK_BD": (217, 119, 6), "HOOK_TX": (124, 65, 16), + "IO_FILL": (241, 245, 249), "IO_BD": (100, 116, 139), "IO_TX": (51, 65, 85), + "PR_FILL": (237, 233, 254), "PR_BD": (124, 58, 237), "PR_TX": (76, 29, 149), + "TRANS_FILL": (248, 250, 252), "TRANS_BD": (203, 213, 225), + "HARN_FILL": (245, 243, 255), "HARN_BD": (124, 58, 237), "HARN_TX": (67, 56, 202), + "LOOP": (220, 38, 38), + "DIAMOND_FILL": (254, 249, 195), "DIAMOND_TX": (133, 77, 14), + "SEP": (229, 210, 170), "HOOKNAME": (146, 64, 14), "HARNDESC": (55, 48, 110), +} +DARK = { + "BG": (13, 17, 23), "LABELBG": (13, 17, 23), + "INK": (230, 237, 243), "GRAY": (148, 158, 170), "LINE": (139, 148, 158), "SLATE": (162, 172, 186), + "SKILL_FILL": (23, 37, 64), "SKILL_BD": (96, 165, 250), "SKILL_TX": (191, 219, 254), + "AGENT_FILL": (18, 46, 33), "AGENT_BD": (74, 222, 128), "AGENT_TX": (187, 247, 208), + "HOOK_FILL": (38, 32, 17), "HOOK_BD": (217, 160, 40), "HOOK_TX": (252, 211, 77), + "IO_FILL": (30, 41, 59), "IO_BD": (113, 128, 150), "IO_TX": (203, 213, 225), + "PR_FILL": (45, 40, 74), "PR_BD": (167, 139, 250), "PR_TX": (221, 214, 254), + "TRANS_FILL": (30, 41, 59), "TRANS_BD": (71, 85, 105), + "HARN_FILL": (35, 31, 64), "HARN_BD": (139, 109, 240), "HARN_TX": (199, 191, 247), + "LOOP": (248, 113, 113), + "DIAMOND_FILL": (60, 52, 20), "DIAMOND_TX": (253, 224, 71), + "SEP": (70, 60, 35), "HOOKNAME": (252, 211, 77), "HARNDESC": (199, 191, 247), +} + +d = None # set per build() + + +# ----------------------------------------------------------------------- helpers +# Helpers read theme colours from globals at CALL time (no colour default args, so +# nothing binds to a single theme at def time). +def box(x0, y0, x1, y1, fill, outline, width=2, radius=10): + d.rounded_rectangle([P(x0), P(y0), P(x1), P(y1)], radius=P(radius), + fill=fill, outline=outline, width=P(width)) + + +def dashed_line(x0, y0, x1, y1, color, width=2, dash=9, gap=6): + import math + dx, dy = x1 - x0, y1 - y0 + dist = math.hypot(dx, dy) + if dist == 0: + return + ux, uy = dx / dist, dy / dist + pos = 0.0 + while pos < dist: + seg = min(dash, dist - pos) + d.line([P(x0 + ux * pos), P(y0 + uy * pos), + P(x0 + ux * (pos + seg)), P(y0 + uy * (pos + seg))], fill=color, width=P(width)) + pos += dash + gap + + +def dashed_rect(x0, y0, x1, y1, color, width=2, dash=10, gap=6): + dashed_line(x0, y0, x1, y0, color, width, dash, gap) + dashed_line(x1, y0, x1, y1, color, width, dash, gap) + dashed_line(x1, y1, x0, y1, color, width, dash, gap) + dashed_line(x0, y1, x0, y0, color, width, dash, gap) + + +def text(x, y, s, size=12.5, color=None, bold=False, anchor="lm"): + d.text((P(x), P(y)), s, font=font(size, bold), fill=color if color is not None else INK, anchor=anchor) + + +def text_bg(x, y, s, size=10.5, color=None, bold=False, bg=None, padx=4, pady=1): + f = font(size, bold) + w = f.getlength(s) / S + h = size * 1.25 + d.rectangle([P(x - w / 2 - padx), P(y - h / 2 - pady), + P(x + w / 2 + padx), P(y + h / 2 + pady)], fill=bg if bg is not None else LABELBG) + d.text((P(x), P(y)), s, font=f, fill=color if color is not None else INK, anchor="mm") + + +def rich(x, y, segs, size=12.5): + cx = x + for t, color, bold in segs: + f = font(size, bold) + d.text((P(cx), P(y)), t, font=f, fill=color, anchor="lm") + cx += f.getlength(t) / S + return cx + + +def arrowhead(x, y, direction, size=7, color=None): + color = color if color is not None else LINE + if direction == "down": + pts = [(x, y), (x - size, y - size * 1.5), (x + size, y - size * 1.5)] + elif direction == "up": + pts = [(x, y), (x - size, y + size * 1.5), (x + size, y + size * 1.5)] + elif direction == "left": + pts = [(x, y), (x + size * 1.5, y - size), (x + size * 1.5, y + size)] + else: + pts = [(x, y), (x - size * 1.5, y - size), (x - size * 1.5, y + size)] + d.polygon([(P(a), P(b)) for a, b in pts], fill=color) + + +def red_dot(x, y, r=4.5, color=None): + d.ellipse([P(x - r), P(y - r), P(x + r), P(y + r)], fill=color if color is not None else LOOP) + + +# -------------------------------------------------------------------- geometry +MX0, MX1 = 60, 642 +MCX = (MX0 + MX1) / 2 +HX0, HX1 = 694, 1162 + + +def build(theme, out): + globals().update(theme) + global d + img = Image.new("RGB", (W * S, H * S), BG) + d = ImageDraw.Draw(img) + + def trans(y0, lines): + n = len(lines) + gap = {1: 56, 2: 70, 3: 86}[n] + y1 = y0 + gap + d.line([P(MCX), P(y0), P(MCX), P(y1)], fill=LINE, width=P(2)) + arrowhead(MCX, y1, "down") + ph = 16 + 17 * n + pw = max(font(10.5).getlength(s) / S for s in lines) + 30 + cy = (y0 + y1) / 2 + box(MCX - pw / 2, cy - ph / 2, MCX + pw / 2, cy + ph / 2, TRANS_FILL, TRANS_BD, width=1.3, radius=9) + ty = cy - (n - 1) * 8.5 + for s in lines: + text(MCX, ty, s, size=10.5, color=SLATE, anchor="mm") + ty += 17 + return y1 + + # header + text(40, 34, "/brief chain=on — execution flow", size=24, bold=True, color=INK) + text(40, 64, "what each phase does, what passes between phases, which harness applies — and where the two loops are", + size=12.5, color=GRAY) + lx = 40 + ly = 92 + + def swatch(x, fill, bd, label, dashed=False): + if dashed: + dashed_line(x, ly, x + 22, ly, LOOP, 2.6, 5, 3) + else: + d.rounded_rectangle([P(x), P(ly - 6), P(x + 20), P(ly + 7)], radius=P(3), + fill=fill, outline=bd, width=P(1.6)) + tx = x + (28 if not dashed else 30) + text(tx, ly, label, size=11, color=INK) + return tx + font(11).getlength(label) / S + 24 + + lx = swatch(lx, SKILL_FILL, SKILL_BD, "skill") + lx = swatch(lx, AGENT_FILL, AGENT_BD, "agent (subagent)") + lx = swatch(lx, HOOK_FILL, HOOK_BD, "hook") + lx = swatch(lx, TRANS_FILL, TRANS_BD, "handoff + harness") + lx = swatch(lx, None, None, "loop / iterates", dashed=True) + + # 1. user IO + box(MX0, 126, MX1, 172, IO_FILL, IO_BD) + text(MCX, 149, 'User types /brief "" (chain=on is the default)', + size=12.5, color=IO_TX, anchor="mm") + d.line([P(MCX), P(172), P(MCX), P(204)], fill=LINE, width=P(2)) + arrowhead(MCX, 204, "down") + + # 2. /brief + B0 = 204 + box(MX0, B0, MX1, B0 + 122, SKILL_FILL, SKILL_BD) + text(MX0 + 16, B0 + 20, "/brief", size=15.5, bold=True, color=SKILL_TX) + text(MX0 + 70, B0 + 21, "· skill", size=11.5, color=GRAY) + rich(MX0 + 16, B0 + 48, [("spawns ", INK, False), ("researcher", AGENT_TX, True), + (" (subagent) — quick codebase pre-scan", INK, False)], size=12) + text(MX0 + 16, B0 + 70, "runs an interactive Socratic interview to capture requirements", size=12, color=INK) + rich(MX0 + 16, B0 + 94, [("writes ", INK, False), ("brief.md", IO_TX, True), + (" + ", GRAY, False), ("autopilot-policy.yaml", IO_TX, True)], size=12) + + y = trans(B0 + 122, ["hands off brief.md · on chain=on, auto-kick.yaml arms the", + "Stop-hook chain so the hand-off cannot stall"]) + + # 3. /create-ticket + C0 = y + box(MX0, C0, MX1, C0 + 150, SKILL_FILL, SKILL_BD) + text(MX0 + 16, C0 + 20, "/create-ticket", size=15.5, bold=True, color=SKILL_TX) + text(MX0 + 142, C0 + 21, "· skill", size=11.5, color=GRAY) + text(MX0 + 16, C0 + 48, "splits the scope into N tickets through independent layers (Agent tool):", size=12, color=INK) + rich(MX0 + 16, C0 + 72, [("researcher", AGENT_TX, True), (" → ", GRAY, False), + ("decomposer", AGENT_TX, True), (" → ", GRAY, False), + ("planner", AGENT_TX, True), (" → ", GRAY, False), + ("ticket-evaluator", AGENT_TX, True)], size=12) + rich(MX0 + 16, C0 + 98, [("quality gate FAIL → ", INK, False), ("planner", AGENT_TX, True), + (" retry ×2 ", INK, False), ("(loop)", LOOP, True)], size=12) + rich(MX0 + 16, C0 + 124, [("writes ", INK, False), ("ticket.md ×N + split-plan.md", IO_TX, True)], size=12) + + y = trans(C0 + 150, ["split-plan.md is the single source of truth · one", + "phase-state.yaml is created per ticket (the state machine)"]) + + # 4. /autopilot + A0 = y + box(MX0, A0, MX1, A0 + 84, SKILL_FILL, SKILL_BD) + text(MX0 + 16, A0 + 22, "/autopilot", size=15.5, bold=True, color=SKILL_TX) + text(MX0 + 104, A0 + 23, "· skill — orchestrator", size=11.5, color=GRAY) + text(MX0 + 16, A0 + 52, "runs each ticket in dependency (topological) order", size=12, color=INK) + + y = trans(A0 + 84, ["Stop hook autopilot-continue re-injects a \"continue\" prompt every", + "turn — this is what DRIVES the loop; state files make the run resumable"]) + + # PER-TICKET LOOP box + OLY0 = y + 4 + OLX0, OLX1 = 40, 666 + text(OLX0 + 16, OLY0 + 18, "PER-TICKET LOOP — one pass per ticket", size=13, bold=True, color=LOOP) + text(OLX0 + 16, OLY0 + 38, "(auto-/compact resets the context window at every ticket boundary)", size=10.5, color=LOOP) + + SX0, SX1 = 64, 638 + SCX = (SX0 + SX1) / 2 + + # 5. /scout + S0 = OLY0 + 60 + box(SX0, S0, SX1, S0 + 118, SKILL_FILL, SKILL_BD) + text(SX0 + 14, S0 + 20, "/scout", size=14.5, bold=True, color=SKILL_TX) + text(SX0 + 76, S0 + 21, "· skill (thin orchestrator) — chains two sub-skills:", size=11, color=GRAY) + rich(SX0 + 14, S0 + 50, [("/investigate", SKILL_TX, True), (" → ", GRAY, False), + ("researcher", AGENT_TX, True), (" → ", GRAY, False), + ("investigation.md", IO_TX, True)], size=12) + rich(SX0 + 14, S0 + 76, [("/plan2doc", SKILL_TX, True), (" → ", GRAY, False), + ("planner", AGENT_TX, True), (" (size-routed) → ", GRAY, False), + ("plan.md", IO_TX, True)], size=12) + + y = trans(S0 + 118, ["investigation.md + plan.md land on disk · the plan's Acceptance", + "Criteria become the FIXED rubric for the evaluator (information firewall)"]) + + # 6. /impl + I0 = y + IMPL_H = 268 + box(SX0, I0, SX1, I0 + IMPL_H, SKILL_FILL, SKILL_BD) + text(SX0 + 14, I0 + 20, "/impl", size=14.5, bold=True, color=SKILL_TX) + text(SX0 + 64, I0 + 21, "· skill — runs the Generator / Evaluator loop, then audits", size=11, color=GRAY) + + ILY0 = I0 + 40 + ILY1 = ILY0 + 132 + dashed_rect(SX0 + 18, ILY0, SX1 - 18, ILY1, LOOP, width=2, dash=8, gap=5) + text(SX0 + 30, ILY0 + 16, "VERIFY LOOP — max 9 rounds (default)", size=10.5, bold=True, color=LOOP) + gA0, gA1 = SX0 + 34, SX0 + 280 + eA0, eA1 = SX1 - 280, SX1 - 34 + gby0, gby1 = ILY0 + 36, ILY0 + 96 + box(gA0, gby0, gA1, gby1, AGENT_FILL, AGENT_BD, radius=8) + text((gA0 + gA1) / 2, gby0 + 22, "implementer", size=12.5, bold=True, color=AGENT_TX, anchor="mm") + text((gA0 + gA1) / 2, gby0 + 42, "(Generator) writes code", size=10, color=AGENT_TX, anchor="mm") + box(eA0, gby0, eA1, gby1, AGENT_FILL, AGENT_BD, radius=8) + text((eA0 + eA1) / 2, gby0 + 22, "ac-evaluator", size=12.5, bold=True, color=AGENT_TX, anchor="mm") + text((eA0 + eA1) / 2, gby0 + 42, "(Evaluator) per-AC verdict", size=10, color=AGENT_TX, anchor="mm") + fy = (gby0 + gby1) / 2 + d.line([P(gA1), P(fy), P(eA0), P(fy)], fill=LINE, width=P(2.4)) + arrowhead(eA0, fy, "right") + text_bg((gA1 + eA0) / 2, fy - 14, "verify", size=10, color=GRAY) + mid = (gA0 + eA1) / 2 + gcx = (gA0 + gA1) / 2 + ecx = (eA0 + eA1) / 2 + by = gby1 + 16 + dashed_line(ecx, gby1, ecx, by, LOOP, 2, 6, 4) + dashed_line(ecx, by, gcx, by, LOOP, 2, 6, 4) + dashed_line(gcx, by, gcx, gby1, LOOP, 2, 6, 4) + arrowhead(gcx, gby1 + 2, "up", color=LOOP) + text_bg(mid, by, "FAIL → next round", size=10.5, color=LOOP, bold=True) + + rich(SX0 + 14, ILY1 + 22, [("on ", INK, False), ("PASS", AGENT_TX, True), (" → ", GRAY, False), + ("/audit", SKILL_TX, True), + (" (skill) spawns three reviewers in parallel:", INK, False)], size=12) + ay0 = ILY1 + 38 + ay1 = ay0 + 48 + aw = (SX1 - SX0 - 28 - 2 * 14) / 3 + for i, (nm, sub) in enumerate([("code-reviewer", None), ("security-scanner", None), + ("doc-verifier *", "(conditional)")]): + ax0 = SX0 + 14 + i * (aw + 14) + box(ax0, ay0, ax0 + aw, ay1, AGENT_FILL, AGENT_BD, radius=8) + if sub: + text(ax0 + aw / 2, ay0 + 18, nm, size=11, bold=True, color=AGENT_TX, anchor="mm") + text(ax0 + aw / 2, ay0 + 34, sub, size=9, color=AGENT_TX, anchor="mm") + else: + text(ax0 + aw / 2, ay0 + 24, nm, size=11, bold=True, color=AGENT_TX, anchor="mm") + + y = trans(I0 + IMPL_H, ["subagents return < 500-token summaries; full artifacts stay on", + "disk (context conservation) — the orchestrator window stays lean"]) + + # 7. /ship + SH0 = y + box(SX0, SH0, SX1, SH0 + 122, SKILL_FILL, SKILL_BD) + text(SX0 + 14, SH0 + 20, "/ship", size=14.5, bold=True, color=SKILL_TX) + text(SX0 + 60, SH0 + 21, "· skill", size=11, color=GRAY) + rich(SX0 + 14, SH0 + 48, [("commit (", INK, False), ("git", IO_TX, True), (") → ", GRAY, False), + ("/tune", SKILL_TX, True), (" distils eval logs into the cross-session KB", INK, False)], size=12) + rich(SX0 + 14, SH0 + 72, [("push branch → open a Pull Request via ", INK, False), + ("gh", IO_TX, True)], size=12) + rich(SX0 + 14, SH0 + 96, [("move ticket → ", INK, False), ("backlog/done/", IO_TX, True)], size=12) + + # decision diamond + DCX, DCY = SCX, SH0 + 122 + 56 + d.line([P(SCX), P(SH0 + 122), P(SCX), P(DCY - 30)], fill=LINE, width=P(2)) + arrowhead(SCX, DCY - 30, "down") + d.polygon([(P(DCX), P(DCY - 28)), (P(DCX + 104), P(DCY)), + (P(DCX), P(DCY + 28)), (P(DCX - 104), P(DCY))], + fill=DIAMOND_FILL, outline=LOOP, width=P(2)) + text(DCX, DCY, "more tickets?", size=11.5, bold=True, color=DIAMOND_TX, anchor="mm") + + OLY1 = DCY + 52 + dashed_rect(OLX0, OLY0, OLX1, OLY1, LOOP, width=2.4, dash=11, gap=6) + + # yes -> loop back up to /scout + LBX = 50 + dashed_line(DCX - 104, DCY, LBX, DCY, LOOP, 2.4, 9, 6) + dashed_line(LBX, DCY, LBX, S0 + 50, LOOP, 2.4, 9, 6) + dashed_line(LBX, S0 + 50, SX0, S0 + 50, LOOP, 2.4, 9, 6) + arrowhead(SX0, S0 + 50, "right", color=LOOP) + text_bg(DCX - 150, DCY, "yes — next ticket", size=10.5, color=LOOP, bold=True) + + # no -> PR + d.line([P(SCX), P(DCY + 28), P(SCX), P(OLY1 + 46)], fill=LINE, width=P(2)) + arrowhead(SCX, OLY1 + 46, "down") + text_bg(SCX + 96, OLY1 + 24, "no (all tickets done)", size=10.5, color=GRAY) + + # PR + PR0 = OLY1 + 48 + box(MX0, PR0, MX1, PR0 + 48, PR_FILL, PR_BD) + text(MCX, PR0 + 24, "All tickets shipped → Pull Request(s) on GitHub", + size=13, bold=True, color=PR_TX, anchor="mm") + + # HOOKS lane + HY0, HY1 = 204, 1196 + box(HX0, HY0, HX1, HY1, HOOK_FILL, HOOK_BD, width=2, radius=12) + text(HX0 + 18, HY0 + 24, "Lifecycle hooks — always-on", size=14, bold=True, color=HOOK_TX) + text(HX0 + 18, HY0 + 45, "(hooks.json — fire around every tool call & every turn)", size=10, color=GRAY) + + def hgroup(yy, title): + text(HX0 + 18, yy, title, size=12, bold=True, color=HOOK_TX) + return yy + 22 + + def hitem(yy, name, desc, marker=False): + if marker: + red_dot(HX0 + 22, yy, 4.5) + rich(HX0 + 32, yy, [(name, HOOKNAME, True)], size=10.5) + if desc: + text(HX0 + 32, yy + 16, desc, size=9.5, color=GRAY) + return yy + 35 + return yy + 22 + + hy = HY0 + 78 + hy = hgroup(hy, "SessionStart") + hy = hitem(hy, "session-start.sh", "boot + resume re-inject") + hy += 8 + hy = hgroup(hy, "PreToolUse") + hy = hitem(hy, "pre-bash / write / edit-safety.sh", "PII · destructive · identity guards") + hy = hitem(hy, "pre-state-transition.sh", "block illegal status writes") + hy = hitem(hy, "pre-bash-contract-guard.sh", "block bash state-mutation") + hy = hitem(hy, "pre-next-scout-auto-compact.sh", "/compact at ticket boundary") + hy = hitem(hy, "pre-askuserquestion-guard.sh", "non-interactive gate") + hy += 8 + hy = hgroup(hy, "PreCompact") + hy = hitem(hy, "pre-compact-save.sh", "snapshot state before /compact") + hy += 8 + hy = hgroup(hy, "PostToolUse") + hy = hitem(hy, "post-phase-checkpoint.sh", "persist phase-state.yaml") + hy = hitem(hy, "post-ship-state-auto-compact.sh", "/compact safety net") + hy = hitem(hy, "post-skill-cleanup.sh", "") + hy += 8 + hy = hgroup(hy, "Stop (the loop drivers)") + ac_y = hy + hy = hitem(hy, "autopilot-continue.sh", "re-injects \"continue\" = the loop", marker=True) + hy = hitem(hy, "impl- / scout-checkpoint-guard.sh", "block premature stop mid-phase") + hy = hitem(hy, "session-stop-log.sh", "") + + hy += 12 + d.line([P(HX0 + 18), P(hy), P(HX1 - 18), P(hy)], fill=SEP, width=P(1)) + hy += 16 + text(HX0 + 18, hy, "How the hooks shape the loop", size=11.5, bold=True, color=HOOK_TX) + hy += 22 + for ln in ["• Pre-*-safety / contract / state guards vet every", + " Write / Edit / Bash before it runs.", + "• auto-compact hooks refresh the context window", + " between tickets so it never fills up.", + "• Stop hooks re-inject \"continue\" until every ticket", + " reaches backlog/done/ — that IS the per-ticket loop."]: + text(HX0 + 26, hy, ln, size=10, color=SLATE) + hy += 17 + + # connector: autopilot-continue -> PER-TICKET LOOP box + gapx = 680 + dashed_line(HX0, ac_y, gapx, ac_y, LOOP, 2.4, 8, 5) + dashed_line(gapx, ac_y, gapx, OLY0 + 26, LOOP, 2.4, 8, 5) + dashed_line(gapx, OLY0 + 26, OLX1, OLY0 + 26, LOOP, 2.4, 8, 5) + arrowhead(OLX1, OLY0 + 26, "left", color=LOOP) + text_bg(OLX0 + 470, OLY0 + 26, "Stop hook drives the loop", size=10.5, color=LOOP, bold=True) + + # HARNESS key panel + KY0 = PR0 + 84 + KY1 = KY0 + 214 + box(MX0, KY0, HX1, KY1, HARN_FILL, HARN_BD, width=2, radius=12) + text(MX0 + 18, KY0 + 24, "The harness applied across the whole run", size=14, bold=True, color=HARN_TX) + text(MX0 + 18, KY0 + 47, "Together these mechanisms ARE the plugin's closed inner loop of loop engineering — the scheduler (\"outer\") loop is delegated to Claude Code (/loop · /schedule).", + size=10.5, color=SLATE) + ky = KY0 + 76 + HARN = [ + ("Information firewall", "code authors and code judges never share a context — the Generator (implementer) and Evaluator (ac-evaluator) are separate fresh subagents, so the judge cannot be biased by the author."), + ("Context conservation", "every subagent returns a < 500-token summary; full artifacts (investigation, plan, eval rounds) live on disk; auto-/compact runs between tickets."), + ("State machine", "autopilot-state.yaml + one phase-state.yaml per ticket record every step, so any compaction, crash or /clear is resumable from the last checkpoint."), + ("Bounded closed loops", "the per-ticket loop and the verify loop (Generator/Evaluator, up to 9 rounds) stop only when the Acceptance-Criteria contract passes or a round cap is hit."), + ("Lifecycle guards", "hooks vet every Write / Edit / Bash, snapshot state before /compact, and decide continue-vs-stop on every turn."), + ] + for title, desc in HARN: + ex = rich(MX0 + 18, ky, [("• ", HARN_TX, True), (title + " — ", HARN_TX, True)], size=11) + text(ex, ky, desc, size=10.5, color=HARNDESC, anchor="lm") + ky += 27 + + text(40, KY1 + 26, "* doc-verifier runs only when a documentation / advertised-interface surface is touched.", + size=10, color=GRAY) + + img.save(out) + print("wrote", out, img.size) + + +build(LIGHT, os.path.join(HERE, "brief-chain-flow.png")) +build(DARK, os.path.join(HERE, "brief-chain-flow-dark.png")) diff --git a/docs/gen-brief-chain-overview.py b/docs/gen-brief-chain-overview.py new file mode 100644 index 0000000..b29132d --- /dev/null +++ b/docs/gen-brief-chain-overview.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Generate the glanceable `/brief chain=on` overview (light + dark). + +A short, landscape, large-text figure for the TOP of README.md — readable at +GitHub's ~860px content width without zooming. The exhaustive figure (agents, +hooks, harness) lives in docs/gen-brief-chain-flow.py and is linked/collapsed +below it. Renders two files for a dark-mode swap: + + docs/brief-chain-overview.png (light) + docs/brief-chain-overview-dark.png (dark) + +Re-generate: python3 docs/gen-brief-chain-overview.py +Glyph rule: Arial lacks ↻ ★ ⇄ ≤ — use only ASCII plus → · — ×. +""" + +import os +from PIL import Image, ImageDraw, ImageFont + +S = 2 +W, H = 960, 640 +HERE = os.path.dirname(os.path.abspath(__file__)) + +_FONTS = { + False: ["/System/Library/Fonts/Supplemental/Arial.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", "/Library/Fonts/Arial.ttf"], + True: ["/System/Library/Fonts/Supplemental/Arial Bold.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", "/Library/Fonts/Arial Bold.ttf"], +} +_fc = {} + + +def font(size, bold=False): + k = (round(size, 1), bold) + if k in _fc: + return _fc[k] + f = None + for p in _FONTS[bold]: + if os.path.exists(p): + try: + f = ImageFont.truetype(p, int(round(size * S))); break + except Exception: + pass + _fc[k] = f or ImageFont.load_default() + return _fc[k] + + +def P(v): + return int(round(v * S)) + + +LIGHT = { + "bg": (255, 255, 255), "ink": (17, 24, 39), "gray": (107, 114, 128), "line": (71, 85, 105), + "skill": ((219, 234, 254), (37, 99, 235), (30, 58, 138)), + "agent": ((220, 252, 231), (22, 163, 74), (20, 83, 45)), + "io": ((241, 245, 249), (100, 116, 139), (51, 65, 85)), + "pr": ((237, 233, 254), (124, 58, 237), (76, 29, 149)), + "loop": (220, 38, 38), "labelbg": (255, 255, 255), +} +DARK = { + "bg": (13, 17, 23), "ink": (230, 237, 243), "gray": (148, 158, 170), "line": (139, 148, 158), + "skill": ((23, 37, 64), (96, 165, 250), (191, 219, 254)), + "agent": ((18, 46, 33), (74, 222, 128), (187, 247, 208)), + "io": ((30, 41, 59), (113, 128, 150), (203, 213, 225)), + "pr": ((45, 40, 74), (167, 139, 250), (221, 214, 254)), + "loop": (248, 113, 113), "labelbg": (13, 17, 23), +} + +d = None +C = None + + +def box(x0, y0, x1, y1, role, width=2.4, radius=11): + fill, bd, _ = C[role] if role in ("skill", "agent", "io", "pr") else (None, role, None) + d.rounded_rectangle([P(x0), P(y0), P(x1), P(y1)], radius=P(radius), fill=fill, outline=bd, width=P(width)) + + +def text(x, y, s, size=12, color=None, bold=False, anchor="lm"): + d.text((P(x), P(y)), s, font=font(size, bold), fill=color or C["ink"], anchor=anchor) + + +def text_bg(x, y, s, size=11, color=None, bold=True): + f = font(size, bold) + w = f.getlength(s) / S + d.rectangle([P(x - w / 2 - 5), P(y - size * 0.75), P(x + w / 2 + 5), P(y + size * 0.75)], fill=C["labelbg"]) + d.text((P(x), P(y)), s, font=f, fill=color or C["ink"], anchor="mm") + + +def rich(x, y, segs, size=12): + cx = x + for t, col, b in segs: + f = font(size, b) + d.text((P(cx), P(y)), t, font=f, fill=col, anchor="lm") + cx += f.getlength(t) / S + return cx + + +def ahead(x, y, dr, size=7, color=None): + color = color or C["line"] + if dr == "down": + p = [(x, y), (x - size, y - size * 1.5), (x + size, y - size * 1.5)] + elif dr == "up": + p = [(x, y), (x - size, y + size * 1.5), (x + size, y + size * 1.5)] + elif dr == "right": + p = [(x, y), (x - size * 1.5, y - size), (x - size * 1.5, y + size)] + else: + p = [(x, y), (x + size * 1.5, y - size), (x + size * 1.5, y + size)] + d.polygon([(P(a), P(b)) for a, b in p], fill=color) + + +def varrow(x, y0, y1, color=None): + d.line([P(x), P(y0), P(x), P(y1)], fill=color or C["line"], width=P(2.4)) + ahead(x, y1, "down", color=color) + + +def harrow(x0, x1, y): + d.line([P(x0), P(y), P(x1), P(y)], fill=C["line"], width=P(2.4)) + ahead(x1, y, "right") + + +def dashed(x0, y0, x1, y1, color, width=2.2, dash=10, gap=6): + import math + dist = math.hypot(x1 - x0, y1 - y0) + if dist == 0: + return + ux, uy = (x1 - x0) / dist, (y1 - y0) / dist + pos = 0.0 + while pos < dist: + seg = min(dash, dist - pos) + d.line([P(x0 + ux * pos), P(y0 + uy * pos), P(x0 + ux * (pos + seg)), P(y0 + uy * (pos + seg))], + fill=color, width=P(width)) + pos += dash + gap + + +def dashed_rect(x0, y0, x1, y1, color, width=2.4, dash=12, gap=7): + dashed(x0, y0, x1, y0, color, width, dash, gap) + dashed(x1, y0, x1, y1, color, width, dash, gap) + dashed(x1, y1, x0, y1, color, width, dash, gap) + dashed(x0, y1, x0, y0, color, width, dash, gap) + + +def swatch(x, y, role, label, dash=False): + if dash: + dashed(x, y, x + 24, y, C["loop"], 2.8, 6, 3) + tx = x + 32 + else: + fill, bd, _ = C[role] + d.rounded_rectangle([P(x), P(y - 7), P(x + 22), P(y + 7)], radius=P(3), fill=fill, outline=bd, width=P(1.8)) + tx = x + 30 + text(tx, y, label, size=12.5, color=C["ink"]) + return tx + font(12.5).getlength(label) / S + 28 + + +def build(theme, out): + global d, C + C = theme + img = Image.new("RGB", (W * S, H * S), C["bg"]) + globals()["d"] = ImageDraw.Draw(img) + + AG = C["agent"][2] + SK = C["skill"][2] + IO = C["io"][2] + GR = C["gray"] + + # header + text(40, 40, "How /brief chain=on runs", size=22, bold=True, color=C["ink"]) + lx = swatch(40, 80, "skill", "skill") + lx = swatch(lx, 80, "agent", "subagent") + lx = swatch(lx, 80, None, "loop", dash=True) + + # ---- band 1: setup chain ---- + bw = (880 - 88) / 3 + xs = [40, 40 + bw + 44, 40 + 2 * (bw + 44)] + by0, by1 = 104, 192 + titles = [("/brief", [[("spawns ", C["ink"], False), ("researcher", AG, True)], + [("writes brief.md + policy", IO, False)]]), + ("/create-ticket", [[("splits scope into N tickets", C["ink"], False)], + [("agents: ", C["ink"], False), ("decomposer, planner", AG, True)]]), + ("/autopilot", [[("orchestrates the run", C["ink"], False)], + [("ticket-by-ticket (topological)", C["ink"], False)]])] + for x0, (nm, lines) in zip(xs, titles): + box(x0, by0, x0 + bw, by1, "skill") + text(x0 + 14, by0 + 24, nm, size=15.5, bold=True, color=SK) + rich(x0 + 14, by0 + 50, lines[0], size=11.5) + rich(x0 + 14, by0 + 70, lines[1], size=11.5) + harrow(xs[0] + bw, xs[1], (by0 + by1) / 2) + harrow(xs[1] + bw, xs[2], (by0 + by1) / 2) + + # autopilot -> loop + apx = xs[2] + bw / 2 + varrow(apx, by1, 224) + + # ---- per-ticket loop container ---- + LX0, LY0, LX1, LY1 = 40, 228, 920, 452 + dashed_rect(LX0, LY0, LX1, LY1, C["loop"]) + text(LX0 + 16, LY0 + 18, "PER-TICKET LOOP — repeats for each ticket", size=13.5, bold=True, color=C["loop"]) + text(LX0 + 16, LY0 + 38, "(auto-/compact resets the context window between tickets)", size=10.5, color=C["loop"]) + + ibw = (848 - 80) / 3 + ix = [56, 56 + ibw + 40, 56 + 2 * (ibw + 40)] + iy0, iy1 = 290, 400 + sc = [("/scout", [[("investigate + plan", C["ink"], False)], + [("researcher · planner", AG, True)]]), + ("/impl", [[("verify loop", C["loop"], True), (" — Generator/Evaluator, max 9", C["ink"], False)], + [("implementer · ac-evaluator", AG, True)]]), + ("/ship", [[("commit + open PR", C["ink"], False)], + [("learns via ", C["ink"], False), ("/tune", SK, True)]])] + for x0, (nm, lines) in zip(ix, sc): + box(x0, iy0, x0 + ibw, iy1, "skill") + text(x0 + 12, iy0 + 22, nm, size=15, bold=True, color=SK) + rich(x0 + 12, iy0 + 48, lines[0], size=11.5) + rich(x0 + 12, iy0 + 70, lines[1], size=11.5) + harrow(ix[0] + ibw, ix[1], (iy0 + iy1) / 2) + harrow(ix[1] + ibw, ix[2], (iy0 + iy1) / 2) + + # loop-back arrow (under boxes) + scx = ix[0] + ibw / 2 + shx = ix[2] + ibw / 2 + lby = 426 + dashed(shx, iy1, shx, lby, C["loop"], 2.4, 9, 6) + dashed(shx, lby, scx, lby, C["loop"], 2.4, 9, 6) + dashed(scx, lby, scx, iy1, C["loop"], 2.4, 9, 6) + ahead(scx, iy1 + 2, "up", color=C["loop"]) + text_bg((scx + shx) / 2, lby, "yes — next ticket", size=10.5, color=C["loop"]) + + # loop -> PR + varrow((LX0 + LX1) / 2, LY1, 488) + text_bg((LX0 + LX1) / 2 + 92, (LY1 + 488) / 2, "all tickets done", size=10, color=GR) + + PX0, PX1 = 320, 640 + box(PX0, 488, PX1, 540, "pr") + text((PX0 + PX1) / 2, 514, "All tickets shipped → Pull Request(s)", size=13, bold=True, color=C["pr"][2], anchor="mm") + + # caption (ties to harness + points to detail) + text(40, 578, "Throughout, lifecycle hooks DRIVE the loop (the Stop hook re-injects \"continue\") and GUARD every write,", + size=11.5, color=GR) + rich(40, 598, [("and each subagent runs in an isolated context (information firewall). ", GR, False), + ("Full flow — agents · hooks · harness — below.", C["ink"], True)], size=11.5) + + img.save(out) + print("wrote", out, img.size) + + +build(LIGHT, os.path.join(HERE, "brief-chain-overview.png")) +build(DARK, os.path.join(HERE, "brief-chain-overview-dark.png")) diff --git a/hooks/accept-set-verify.sh b/hooks/accept-set-verify.sh new file mode 100755 index 0000000..271b193 --- /dev/null +++ b/hooks/accept-set-verify.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# accept-set-verify.sh — PostToolUse(Write|Edit) hook. +# +# Deterministic post-hoc conformance gate over the PERSISTED `## Accept-set +# sweep` section of an `eval-round-{n}.md` report — the Advertised-Accept-Set +# Conformance (AASC) observability line written by the ac-evaluator's +# Persistence-First terminal rewrite (see agents/ac-evaluator.md +# `## Persistence-First Protocol` and the `## Failure-class panel` L-ROBUSTNESS +# lens, plus skills/impl/references/accept-set-conformance-harness.md). +# +# WHY A HOOK (the only recognition-independent lever): the AASC sweep and its +# self-incrimination rule are ALREADY normative MUST in the ac-evaluator lens +# ("on a `triggered-on=` AC, an `## Accept-set sweep` line with `ran=n`, or with +# `ran=y astral=n`, is a NON-CONFORMANT shallow sweep"). Yet because that rule +# is honoured only by the model RECOGNISING it, live dogfoods leaked it +# run-to-run — a triggered boundary stood the sweep down on one re-run of a +# subject it had swept on a prior run. This hook reads the EMITTED line and +# applies the SAME rule deterministically, with zero model recall, closing the +# run-to-run variance that prose alone could not. +# +# WHAT IT CANNOT DO (the bound, R1): it verifies the STRUCTURE of the emitted +# line (a triggered boundary not run; a shallow run that skipped the astral +# complement; a sliced corpus; an authoritative divergence that did not drive +# the verdict to FAIL). It CANNOT verify that a `ran=y astral=y corpus-size=750` +# line is TRUTHFUL — a model could emit those fields without executing the +# sweep. Catching structural self-incrimination, not fabrication, is the ceiling. +# +# Predicates (per `boundary=` line in the section): +# P1 stand-down (all axes): triggered=y ran=n -> BLOCK +# P2 shallow (A,U axes): triggered=y ran=y astral=n -> BLOCK +# P4 gating (all axes): authoritative=y divergences>0 while Status not FAIL/FAIL-CRITICAL -> BLOCK +# P3 thin-corpus (A,U axes): triggered=y ran=y corpus-size < FLOOR -> ADVISORY ONLY (never blocks) +# P3 is ADVISORY, not a gate (dogfood51): corpus-size is a weak proxy for sweep +# depth — a handful of astral probes can be deeper than hundreds of ASCII ones — +# so flooring it would false-trip a legitimately-thin-but-conformant sweep on a +# wide-spec subject (002-r1 A=5, 003-r2 A=105, both astral=y divergences=0, one a +# PASS report). astral (P2) is the real A/U depth gate; P3 only SURFACES the +# run-to-run depth variance as a note. SW_AASC_CORPUS_FLOOR tunes the threshold. +# The `caveat=no-runnable-artifact` escape (a compiled language / no exec +# harness: triggered but the sweep could not be executed) exempts P1/P2 (and the +# P3 advisory) for that line — a legit fail-OPEN degradation. P4 still applies. +# P2/P3 are scoped to the alphabet (A) and unicode-transform (U) axes; the keyed +# (K) and canonical-writer (W) axes legitimately enumerate a small reflection- +# derived corpus (reserved/accessor/private-slot names — a handful of ASCII +# identifiers, never astral), so the astral/corpus checks do not apply to them. +# P1 and P4 are axis-independent. The `## Accept-set sweep` header is matched +# case-insensitively AND at any hash depth (`#`..`######`), so neither a mis-cased +# nor a mis-leveled header can let a whole report silently skip the gate. +# +# Kill switch SW_ACCEPT_SET_CONFORMANCE_MODE: +# on (DEFAULT, v8.5.0 promotion) -> enforce: emit a PostToolUse `decision: block` with the reason. +# metric-only -> observe: log `[ACCEPT-SET-VERIFY] metric-only: would block ...`, ALLOW. +# off -> explicit opt-out: silent skip. +# unknown -> collapses to metric-only (the default is `on`, so the var is set +# ONLY to downgrade; a typo on a downgrade value falls back to the +# safe observe mode — never a surprise enforce, never a silent disable). +# +# Corpus floor SW_AASC_CORPUS_FLOOR (default 256) — the P3 ADVISORY threshold for +# an A/U-axis triggered+run sweep; a thinner corpus is NOTED (never blocked), and +# the threshold is env-tunable per host/subject. +# +# Fail-OPEN iron rule: this hook MUST NEVER break the host Write/Edit. jq +# missing, an unreadable report, a non-eval path, a skeleton (IN_PROGRESS) write, +# the `n/a` fallback, or any internal error is a silent `exit 0`. The exit code +# is ALWAYS 0; the only non-allow influence is the `on`-mode `decision: block` +# JSON, which surfaces the reason to the model (the report write itself already +# completed). + +set -euo pipefail + +# jq is required to parse the PostToolUse payload. Silent skip if missing — +# Write/Edit must never be blocked by this hook (fail-OPEN). +command -v jq >/dev/null 2>&1 || exit 0 + +INPUT=$(cat 2>/dev/null || echo '{}') + +# Gate 1: path match — only AASC eval-round reports. The single glob +# `*eval-round-*.md` covers eval-round-{n}.md, the -part-{i} / -v{i} partition +# and multi-verifier variants, and the docs/eval-round/{topic}-eval-round-{n}.md +# fallback (case globs are not path-aware, so the leading `*` spans directories). +TOOL_FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // ""' 2>/dev/null || echo "") +case "$TOOL_FILE_PATH" in + *eval-round-*.md) ;; + *) exit 0 ;; +esac + +# Gate 2: read the just-written file from disk (NOT the tool payload — the +# terminal rewrite may arrive as a Write of the full body or as Edits, and the +# on-disk state after PostToolUse is the authoritative final content). +[ -f "$TOOL_FILE_PATH" ] || exit 0 + +# Gate 2a: skeleton false-trip guard. The ac-evaluator does TWO writes — a +# `## Status: IN_PROGRESS` skeleton (no sweep section) FIRST, then a terminal +# rewrite. The first `## Status:` line is authoritative (agents/ac-evaluator.md). +# An IN_PROGRESS file is work-in-flight: pass. +FIRST_STATUS=$(grep -m1 -E '^## Status:' "$TOOL_FILE_PATH" 2>/dev/null | sed -E 's/^## Status:[[:space:]]*//' || true) +case "$FIRST_STATUS" in + IN_PROGRESS*) exit 0 ;; +esac + +# Gate 2b: no `## Accept-set sweep` section -> nothing persisted to verify +# (sweep not in scope, or a pre-v8.5.0 / degraded report). Fail-OPEN. Matched +# case-insensitively so a mis-cased header (`## Accept-set Sweep`) cannot let a +# whole report skip the gate (dogfood51: 001-r2 used a capital-S header). +grep -qiE '^#{1,6}[[:space:]]+accept-set sweep[[:space:]]*$' "$TOOL_FILE_PATH" 2>/dev/null || exit 0 + +# Gate 3: kill switch. Unknown collapses to metric-only. +MODE_RAW="${SW_ACCEPT_SET_CONFORMANCE_MODE:-on}" +case "$MODE_RAW" in + on|metric-only|off) MODE="$MODE_RAW" ;; + *) MODE="metric-only" ;; +esac +[ "$MODE" = "off" ] && exit 0 + +CORPUS_FLOOR="${SW_AASC_CORPUS_FLOOR:-256}" +case "$CORPUS_FLOOR" in + ''|*[!0-9]*) CORPUS_FLOOR=256 ;; +esac + +# Extract the `## Accept-set sweep` section body (heading exclusive, up to the +# next `## ` heading or EOF). +SECTION=$(awk ' + tolower($0) ~ /^#{1,6}[[:space:]]+accept-set sweep[[:space:]]*$/ { f=1; next } + /^#{1,6}[[:space:]]/ { f=0 } + f +' "$TOOL_FILE_PATH" 2>/dev/null || true) + +# `n/a` fallback (no external-input boundary in scope) -> conformant by contract. +case "$SECTION" in + *"n/a (no external-input boundary in scope)"*) exit 0 ;; +esac + +# Is the terminal verdict a FAIL? FAIL and FAIL-CRITICAL both start FAIL; PASS +# and PASS-WITH-CAVEATS start PASS. Used by P4. +case "$FIRST_STATUS" in + FAIL*) STATUS_IS_FAIL=1 ;; + *) STATUS_IS_FAIL=0 ;; +esac + +# field_of -> the token value (empty if absent). +field_of() { + local v + v=$(printf '%s\n' "$1" | grep -oE "$2=[^[:space:]]+" | head -1) || true + printf '%s' "${v#*=}" +} + +BLOCKING="" +ADVISORY="" +while IFS= read -r LINE; do + # Select any sweep line carrying a boundary= token (field_of is order-independent, + # so a reordered-but-complete line is no longer skipped). The section is already + # bounded to the `## Accept-set sweep` block by the awk extractor above. + case "$LINE" in + *boundary=*) ;; + *) continue ;; + esac + B=$(field_of "$LINE" boundary) + # Off-grammar boundary label (not A/U/W/K): the A/U astral-depth gate is keyed to + # A|U, so a relabeled alphabet boundary would silently dodge it. Surface it on + # stderr (fail-toward-observability; not a block, to avoid false-tripping a genuine + # W/K-class sweep) so format drift in the only recognition-independent lever is seen. + case "$B" in + A|U|W|K) ;; + "") ;; + *) printf '[ACCEPT-SET-VERIFY] off-grammar boundary label=%s in %s\n' "$B" "$(basename "$TOOL_FILE_PATH")" >&2 ;; + esac + TRIG=$(field_of "$LINE" triggered) + RAN=$(field_of "$LINE" ran) + ASTRAL=$(field_of "$LINE" astral) + CORPUS=$(field_of "$LINE" corpus-size) + DIV=$(field_of "$LINE" divergences) + AUTH=$(field_of "$LINE" authoritative) + CAVEAT=$(field_of "$LINE" caveat) + # corpus-size may carry a descriptive suffix (dogfood51: `10-non-ascii-decimal`); + # take the leading integer so an annotation cannot dodge the P3 advisory. + CORPUS_INT="${CORPUS%%-*}" + + REASON="" + # The no-runnable-artifact caveat is the documented fail-OPEN escape (a + # compiled language / no exec harness: the boundary was triggered but the + # sweep could not be executed). It exempts the execution + depth predicates + # P1/P2 (and the P3 advisory); the gating-consistency predicate P4 still applies. + if [ "$CAVEAT" != "no-runnable-artifact" ]; then + # P1 stand-down (all axes): a triggered boundary that was not run -> BLOCK. + if [ "$TRIG" = "y" ] && [ "$RAN" = "n" ]; then + REASON="P1-stand-down" + elif [ "$TRIG" = "y" ] && [ "$RAN" = "y" ]; then + # P2 shallow astral (A,U axes): astral complement skipped -> BLOCK (astral + # is the real A/U depth gate; the planes are the mandated alphabet complement). + if { [ "$B" = "A" ] || [ "$B" = "U" ]; } && [ "$ASTRAL" = "n" ]; then + REASON="P2-shallow-astral" + # P3 thin corpus (A,U axes): corpus below the floor -> ADVISORY ONLY, never + # blocks (corpus-size is a weak depth proxy; flooring it false-trips a + # legitimately-thin conformant sweep on a wide-spec subject — dogfood51). + elif { [ "$B" = "A" ] || [ "$B" = "U" ]; } \ + && printf '%s' "$CORPUS_INT" | grep -qE '^[0-9]+$' \ + && [ "$CORPUS_INT" -lt "$CORPUS_FLOOR" ]; then + ADVISORY="${ADVISORY}boundary=${B} triggered=${TRIG} ran=${RAN} astral=${ASTRAL} corpus-size=${CORPUS} note=P3-thin-corpus(<$CORPUS_FLOOR)"$'\n' + fi + fi + fi + # P4 gating-consistency (all axes): an authoritative boundary with divergences + # must drive the verdict to FAIL -> BLOCK. + if [ -z "$REASON" ] && [ "$AUTH" = "y" ] \ + && printf '%s' "$DIV" | grep -qE '^[0-9]+$' \ + && [ "$DIV" -gt 0 ] && [ "$STATUS_IS_FAIL" = "0" ]; then + REASON="P4-gating-inconsistency" + fi + + [ -n "$REASON" ] || continue + BLOCKING="${BLOCKING}boundary=${B} triggered=${TRIG} ran=${RAN} astral=${ASTRAL} corpus-size=${CORPUS} reason=${REASON}"$'\n' +done < silent pass. +[ -n "$BLOCKING$ADVISORY" ] || exit 0 + +BASENAME=$(basename "$TOOL_FILE_PATH") + +# Advisory notes (P3 thin-corpus) are emitted in BOTH modes and NEVER block — +# corpus depth is subject-dependent, so this is observability, not a gate. +if [ -n "$ADVISORY" ]; then + while IFS= read -r V; do + [ -n "$V" ] || continue + echo "[ACCEPT-SET-VERIFY] advisory: $BASENAME $V" >&2 + done < exit 0 (an advisory-only report is not a block). +[ -n "$BLOCKING" ] || exit 0 + +if [ "$MODE" = "metric-only" ]; then + while IFS= read -r V; do + [ -n "$V" ] || continue + echo "[ACCEPT-SET-VERIFY] metric-only: would block (file=$BASENAME $V)" >&2 + done <&2 +done </dev/null) || ACTIVE_STEPS=0 +# --- Check for unfinished steps (WI-3 schema-tolerant) --- +# parse_active_steps (hooks/lib/parse-state-file.sh) emits one `:` +# line per in_progress/pending step across all tickets, tolerating the flat +# (`scout: in_progress`), inline-flow (`steps: {scout: in_progress, …}`), and +# nested (`scout:\n status: in_progress`) shapes. The prior inline grep matched +# only the flat/flow shapes and silently stranded a nested-form pipeline. +ACTIVE_STEP_LINES=$(parse_active_steps "$STATE_FILE" 2>/dev/null || true) +# grep -c exits 1 (and prints 0) when there are no active steps; guard it. +ACTIVE_STEPS=$(printf '%s' "$ACTIVE_STEP_LINES" | grep -c ':') || ACTIVE_STEPS=0 if [ "$ACTIVE_STEPS" -eq 0 ]; then # All step-level work done — pipeline is finished, allow stop. @@ -428,10 +437,11 @@ if [ "$ACTIVE_STEPS" -eq 0 ]; then fi # --- Determine next step to execute --- -# Priority: first in_progress step, then first pending step -NEXT_STEP=$(grep -E '(create-ticket|scout|impl|ship): in_progress' "$STATE_FILE" | head -1 | sed 's/^ *//; s/: in_progress//') || true +# Priority: first in_progress step, then first pending step. Derived from the +# same WI-3-tolerant parse_active_steps output (`:` lines). +NEXT_STEP=$(printf '%s\n' "$ACTIVE_STEP_LINES" | grep ':in_progress$' | head -1 | sed 's/:in_progress$//') || true if [ -z "$NEXT_STEP" ]; then - NEXT_STEP=$(grep -E '(create-ticket|scout|impl|ship): pending' "$STATE_FILE" | head -1 | sed 's/^ *//; s/: pending//') || true + NEXT_STEP=$(printf '%s\n' "$ACTIVE_STEP_LINES" | grep ':pending$' | head -1 | sed 's/:pending$//') || true fi NEXT_STEP="${NEXT_STEP:-unknown}" diff --git a/hooks/hooks.json b/hooks/hooks.json index bf57c66..bead92e 100644 --- a/hooks/hooks.json +++ b/hooks/hooks.json @@ -61,6 +61,14 @@ { "matcher": "Edit", "hooks": [{ "type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/post-ship-state-auto-compact.sh" }] + }, + { + "matcher": "Write", + "hooks": [{ "type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/accept-set-verify.sh" }] + }, + { + "matcher": "Edit", + "hooks": [{ "type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/accept-set-verify.sh" }] } ], "Stop": [ diff --git a/hooks/lib/audit-coverage.sh b/hooks/lib/audit-coverage.sh index afd115f..af6b843 100644 --- a/hooks/lib/audit-coverage.sh +++ b/hooks/lib/audit-coverage.sh @@ -127,6 +127,8 @@ audit_coverage_emit() { # one that will be committed, not the on-disk file contents). local entries="" local count=0 + # 'line' is an unused read placeholder; the other names are consumed in the loop below. + # shellcheck disable=SC2034 local line raw_status path_new path_old status_letter blob # Use a process substitution to avoid the subshell trap of `| while`. while IFS=$'\t' read -r raw_status path_new path_old; do @@ -277,6 +279,8 @@ if isinstance(files, list): PYEOF ) || py_out="" if [ -n "$py_out" ]; then + # 'py_line' is an unused read placeholder; the other names are consumed below. + # shellcheck disable=SC2034 local py_line py_kind py_a py_b py_c while IFS=$'\t' read -r py_kind py_a py_b py_c; do case "$py_kind" in @@ -361,6 +365,8 @@ PYEOF local total_cov=${#cov_paths[@]} # Index coverage entries by path for lookup. + # Reserved lookup placeholders, intentionally unassigned here. + # shellcheck disable=SC2034 local cov_blob_for cov_status_for local idx=0 declare -A cov_idx=() @@ -370,6 +376,8 @@ PYEOF done # 1. For each commit-side file: must be covered AND blob must match. + # 'reason' is reserved for future diagnostics; the others are consumed below. + # shellcheck disable=SC2034 local commit_file head_blob audit_blob audit_status reason local short_audit short_head if [ -n "$commit_changes" ]; then @@ -399,6 +407,8 @@ PYEOF idx=0 while [ "$idx" -lt "$total_cov" ]; do audit_blob="${cov_blobs[$idx]}" + # 'audit_status' is captured for symmetry/future use; not read on this path. + # shellcheck disable=SC2034 audit_status="${cov_statuses[$idx]}" local cov_path="${cov_paths[$idx]}" head_blob=$(git rev-parse "HEAD:$cov_path" 2>/dev/null) || head_blob="__deleted__" diff --git a/hooks/lib/parse-state-file.sh b/hooks/lib/parse-state-file.sh index 4a096df..4c09689 100755 --- a/hooks/lib/parse-state-file.sh +++ b/hooks/lib/parse-state-file.sh @@ -645,6 +645,114 @@ PY return 0 } +# --------------------------------------------------------------------------- +# Public function: parse_active_steps +# Usage: parse_active_steps +# +# Emits one line per pipeline step whose status is `in_progress` or `pending`, +# across every ticket, in document order, formatted as `:` +# (e.g. `scout:in_progress`). The autopilot Stop hook +# (`hooks/autopilot-continue.sh`) consumes this to decide whether unfinished +# step-level work remains (line count) and which step runs next (first +# `in_progress`, else first `pending`). +# +# WI-3 schema-tolerance: accepts the canonical-flat (`scout: in_progress`), +# inline-flow (`steps: {scout: in_progress, ...}`), and nested +# (`scout:\n status: in_progress`) step shapes — the same forms the +# auto-compact path tolerates — so a model schema slip does not silently +# strand the continuation driver. Returns 1 only on a missing/empty file +# argument; an empty step set is a normal rc=0 result. +# --------------------------------------------------------------------------- +parse_active_steps() { + local file="$1" + if [ -z "$file" ] || [ ! -f "$file" ]; then + return 1 + fi + + if _psf_have yq; then + yq -r ' + .tickets[].steps // {} | to_entries[] | + select((.value.status // .value) == "in_progress" + or (.value.status // .value) == "pending") | + .key + ":" + (.value.status // .value) + ' "$file" 2>/dev/null + return 0 + fi + + if _psf_have python3 && python3 -c 'import yaml' >/dev/null 2>&1; then + python3 - "$file" <<'PY' 2>/dev/null +import sys +import yaml +with open(sys.argv[1], "r", encoding="utf-8") as fh: + doc = yaml.safe_load(fh) or {} +tickets = doc.get("tickets") +if isinstance(tickets, dict): + entries = list(tickets.values()) +elif isinstance(tickets, list): + entries = tickets +else: + entries = [] +for entry in entries: + if not isinstance(entry, dict): + continue + steps = entry.get("steps") or {} + if not isinstance(steps, dict): + continue + for key, val in steps.items(): + st = val.get("status") if isinstance(val, dict) else val + if st in ("in_progress", "pending"): + print("%s:%s" % (key, st)) +PY + return 0 + fi + + # awk fallback: stateful walk, POSIX awk only. Handles flat + # (`scout: in_progress`), inline-flow (`steps: {scout: in_progress, ...}`), + # and nested (`scout:\n status: in_progress`) step shapes. Only the four + # known step keys (create-ticket|scout|impl|ship) are matched directly, so + # there is no collision with the ticket-level `status:` field; the nested + # `status:` line is consumed only while a step opener is pending (cur_key). + awk ' + BEGIN { in_tickets = 0; cur_key = "" } + /^tickets:[[:space:]]*$/ { in_tickets = 1; next } + in_tickets && /^[^[:space:]-]/ { in_tickets = 0; cur_key = "" } + in_tickets && /^[[:space:]]+steps:[[:space:]]*\{.*\}[[:space:]]*$/ { + line = $0 + sub(/^[[:space:]]+steps:[[:space:]]*\{/, "", line) + sub(/\}[[:space:]]*$/, "", line) + n = split(line, pairs, /,/) + for (i = 1; i <= n; i++) { + kv = pairs[i]; ci = index(kv, ":") + if (ci > 0) { + k = substr(kv, 1, ci - 1); v = substr(kv, ci + 1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", k) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", v) + if (v == "in_progress" || v == "pending") print k ":" v + } + } + cur_key = "" + next + } + in_tickets && /^[[:space:]]+(create-ticket|scout|impl|ship):/ { + line = $0; sub(/^[[:space:]]+/, "", line); ci = index(line, ":") + k = substr(line, 1, ci - 1); v = substr(line, ci + 1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", v) + sub(/[[:space:]]+#.*$/, "", v) + if (v == "") { cur_key = k } + else { if (v == "in_progress" || v == "pending") print k ":" v; cur_key = "" } + next + } + in_tickets && cur_key != "" && /^[[:space:]]+status:[[:space:]]*/ { + line = $0; sub(/^[[:space:]]+status:[[:space:]]*/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + if (line == "in_progress" || line == "pending") print cur_key ":" line + cur_key = "" + next + } + ' "$file" + return 0 +} + # --------------------------------------------------------------------------- # Public function: find_phase_state_file # Usage: find_phase_state_file [start_dir] diff --git a/hooks/lib/state-authority.sh b/hooks/lib/state-authority.sh index 04a5519..d087036 100644 --- a/hooks/lib/state-authority.sh +++ b/hooks/lib/state-authority.sh @@ -49,9 +49,16 @@ # tests), source the lib in a subshell with a controlled registry override. # --------------------------------------------------------------------------- -# Registry: hook-owned fields (shipped empty — Negative AC-2 enforced). +# Registry: hook-owned fields (Foundation 3 — proposal 4 / ST-03). +# `.runtime_metrics` is an append-only telemetry list written EXCLUSIVELY by the +# autopilot Stop / PreCompact / checkpoint hooks (see hooks/lib/runtime-metrics.sh +# header). Registering it lets pre-write/pre-edit-safety detect a model full-file +# Write/Edit that would clobber hook-appended entries (the ST-03 lost-update). +# The actual DENY is gated by SW_STATE_FIELD_GUARD_MODE in those callers +# (default `metric-only` — observe, do not block — so this enforcement ships +# opt-in and the prior no-op behaviour is the default until promoted to `on`). # --------------------------------------------------------------------------- -declare -A HOOK_OWNED_FIELDS=() +declare -A HOOK_OWNED_FIELDS=([".runtime_metrics"]=1) # --------------------------------------------------------------------------- # Internal helpers (not part of the public contract; prefix _sa_). @@ -256,8 +263,11 @@ is_hook_owned_field() { for reg_key in "${!HOOK_OWNED_FIELDS[@]}"; do # Convert * to +([!.]) so a single * matches one segment with no dots. pat=$(printf '%s' "$reg_key" | sed 's/\*/+([!.])/g') + # $pat is intentionally UNQUOTED in the case arm below — the extglob pattern must + # be expanded as a glob to match `key`; quoting it would break is_hook_owned_field. + # shellcheck disable=SC2254 case "$key" in - $pat) result=0; break ;; # UNQUOTED $pat — required for glob expansion + $pat) result=0; break ;; esac done eval "$_prev" @@ -268,6 +278,8 @@ is_hook_owned_field() { # Public function: state_field_change_blocked # --------------------------------------------------------------------------- state_field_change_blocked() { + # 'state_file' is part of the public signature but unused on this path. + # shellcheck disable=SC2034 local state_file="$1" local old_string="$2" local new_string="$3" @@ -300,8 +312,10 @@ state_field_change_blocked() { # F-BLANK: block whenever old_val is non-empty AND new_val differs, # including blank-out (new_val empty). Initial-set is already handled - # by the early-exit above. + # by the early-exit above. Echo the matched registry key so the caller + # can name the violated field in its block reason (proposal 4 / UX-10). if [ "$old_val" != "$new_val" ]; then + printf '%s\n' "$reg_key" return 0 fi done diff --git a/hooks/post-ship-state-auto-compact.sh b/hooks/post-ship-state-auto-compact.sh index c8dbfa6..30d4ec8 100755 --- a/hooks/post-ship-state-auto-compact.sh +++ b/hooks/post-ship-state-auto-compact.sh @@ -105,8 +105,11 @@ esac TOOL_PAYLOAD=$(echo "$INPUT" | jq -r '.tool_input.new_string // .tool_input.content // ""' 2>/dev/null || echo "") _detect_ship_completed_in_payload() { local payload="$1" - # Flat form: same line "ship: completed" - if printf '%s' "$payload" | grep -qE '(^|[[:space:]])ship:[[:space:]]+completed([[:space:]]|$)'; then + # Flat form: same line "ship: completed". The trailing class also accepts the + # inline-flow shape `steps: {…, ship: completed}` / `ship: completed,` (ST-11): + # `}` and `,` are valid YAML flow-mapping terminators, so without them the gate + # silently missed a template-seeded flow-form ship-completed write. + if printf '%s' "$payload" | grep -qE '(^|[[:space:]])ship:[[:space:]]+completed([[:space:],}]|$)'; then return 0 fi # Nested form: a line ending in `ship:` followed within 4 lines by @@ -182,6 +185,13 @@ while IFS= read -r TICKET_DIR; do */backlog/active/*) RESOLVED_TICKET_DIR="${RESOLVED_TICKET_DIR//\/backlog\/active\//\/backlog\/done\/}" ;; + */backlog/product_backlog/*) + # ST-02: a never-scouted ticket whose ticket_dir still holds the initial + # product_backlog form would otherwise pass the -d check (the dir exists in + # product_backlog/) and let a fabricated ship: completed inject. Rewrite to + # the done/ form so the existence check catches the state-lie. + RESOLVED_TICKET_DIR="${RESOLVED_TICKET_DIR//\/backlog\/product_backlog\//\/backlog\/done\/}" + ;; esac if [ ! -d "$RESOLVED_TICKET_DIR" ]; then echo "[POST-SHIP-STATE-AUTO-COMPACT] state-lie protection: ticket dir not in done/ ($RESOLVED_TICKET_DIR). Skipping inject — model wrote ship: completed without actually completing ship body." >&2 @@ -260,6 +270,11 @@ if [ "$PSI_MODE" != "off" ]; then */backlog/active/*) PSI_RESOLVED="${PSI_RESOLVED//\/backlog\/active\//\/backlog\/done\/}" ;; + */backlog/product_backlog/*) + # ST-02: same product_backlog-form rewrite as Gate 5 so the per-ticket + # phase-state.yaml lookup resolves to the done/ location. + PSI_RESOLVED="${PSI_RESOLVED//\/backlog\/product_backlog\//\/backlog\/done\/}" + ;; esac # Strip any trailing slash so we can append /phase-state.yaml uniformly. PSI_RESOLVED="${PSI_RESOLVED%/}" diff --git a/hooks/pre-bash-contract-guard.sh b/hooks/pre-bash-contract-guard.sh index 7141a37..54e8216 100755 --- a/hooks/pre-bash-contract-guard.sh +++ b/hooks/pre-bash-contract-guard.sh @@ -160,5 +160,51 @@ if printf '%s' "$COMMAND" | grep -qE "$GIT_COMMIT_RE"; then fi fi +# --------------------------------------------------------------------------- +# Detection 3 (knob-gated, M-03 / ST-04): Bash-mediated state-file status +# mutation. The skip-transition guard (pre-state-transition.sh) is wired only +# to Write/Edit, so a model blocked there can reroute to a raw `yq -i` / +# `sed -i` / shell-redirect on autopilot-state.yaml / phase-state.yaml and flip +# a `status:` to skipped/completed unobserved (the autopilot Stop hook then +# honours the Bash-written transition as pipeline progress). +# +# This NEW deny ships OFF-by-default. It is gated by SW_BASH_STATE_GUARD_MODE so +# the behaviour change is opt-in (matches the repo's metric-only-first rollout +# convention for intrusive guards): +# on -> emit decision:block (deny) +# metric-only -> (default) log `[PRE-BASH-CONTRACT-GUARD] metric-only: would +# deny ...` to stderr and ALLOW the call +# off -> skip detection 3 entirely +# Detections 1 and 2 above are NOT gated by this knob (the hook's NAC posture is +# preserved). A typo collapses to metric-only (the observe-only default). +STATE_GUARD_MODE="${SW_BASH_STATE_GUARD_MODE:-metric-only}" +if [ "$STATE_GUARD_MODE" != "off" ] \ + && printf '%s' "$COMMAND" | grep -qE '(autopilot-state|phase-state)\.yaml'; then + _sg_mutation=false + # `yq -i` / `sed -i` (with any flags/args before -i), or a `>` / `>>` redirect + # whose target is a state file. + if printf '%s' "$COMMAND" | grep -qE '(^|[|;&[:space:]])(yq|sed)([[:space:]]+[^|;&]*)?[[:space:]]-i([[:space:]]|=|$)'; then + _sg_mutation=true + fi + if printf '%s' "$COMMAND" | grep -qE '>>?[[:space:]]*[^[:space:]|;&]*(autopilot-state|phase-state)\.yaml'; then + _sg_mutation=true + fi + # A step/ticket status being driven to a terminal/skip value. The + # `[^[:alpha:]]{0,15}` gap tolerates `= "..."`, `: ...`, `= \"...\"`, `[].`. + _sg_transition=false + if printf '%s' "$COMMAND" | grep -qiE '(status|steps|ship|scout|impl|create-ticket)[^[:alpha:]]{0,15}(skipped|completed|failed|in_progress|in-progress)'; then + _sg_transition=true + fi + if [ "$_sg_mutation" = true ] && [ "$_sg_transition" = true ]; then + case "$STATE_GUARD_MODE" in + on) + emit_block "unauthorized_state_mutate_bash" \ + "Direct Bash mutation of a state-file status (yq -i / sed -i / redirect on autopilot-state.yaml / phase-state.yaml) bypasses the Write/Edit skip-transition guard (pre-state-transition.sh). Route state transitions through the owning Skill (/scout, /impl, /ship) or a Write/Edit so the hook-layer guards fire. Field reference: docs/state-schema.md; see also skills/autopilot/SKILL.md." ;; + metric-only|*) + printf '[PRE-BASH-CONTRACT-GUARD] metric-only: would deny unauthorized_state_mutate_bash (Bash state-file status mutation): %s\n' "${COMMAND:0:140}" >&2 ;; + esac + fi +fi + # All checks passed -> allow. exit 0 diff --git a/hooks/pre-bash-safety.sh b/hooks/pre-bash-safety.sh index bc38cb8..4a3aeed 100755 --- a/hooks/pre-bash-safety.sh +++ b/hooks/pre-bash-safety.sh @@ -1,12 +1,39 @@ #!/usr/bin/env bash set -euo pipefail + +# jq is a documented hard dependency. When it is missing this guard cannot parse +# the tool payload; rather than dying with a silent `exit 127` (which Claude Code +# treats as a non-blocking error — a fail-OPEN), resolve the behaviour through +# SW_SAFETY_JQ_MISSING_MODE (UX-11): +# on -> fail CLOSED (exit 2) with an explicit message +# metric-only -> (default) log the would-be fail-closed and ALLOW (exit 0) +# off -> silently allow (exit 0) +# A typo collapses to metric-only (the observe-only default), never to a silent +# fail-open. Default is metric-only so this hardening does not change the shipped +# fail-open behaviour until an operator opts in with `=on` (post-dogfood promote). +if ! command -v jq >/dev/null 2>&1; then + case "${SW_SAFETY_JQ_MISSING_MODE:-metric-only}" in + on) + echo "[SAFETY-JQ-MISSING] pre-bash-safety: jq not found on PATH — failing closed (exit 2). Install jq (e.g. 'brew install jq') to run this guard." >&2 + exit 2 ;; + off) + exit 0 ;; + metric-only|*) + echo "[SAFETY-JQ-MISSING] metric-only: pre-bash-safety would fail closed (exit 2) — jq not found on PATH; allowing this call. Install jq, or set SW_SAFETY_JQ_MISSING_MODE=on to enforce." >&2 + exit 0 ;; + esac +fi + INPUT=$(cat) COMMAND=$(echo "$INPUT" | jq -r '.tool_input.command // empty') # --- Destructive command patterns --- # Detect at any token position: start, after pipe, after semicolon, after &&/|| # Optional env/command prefix before the actual destructive command -DESTRUCTIVE='(^|[|;&]|\$\(|`)\s*(env\s+|command\s+)?(rm\s+-[A-Za-z]*[rR][A-Za-z]*f|rm\s+-[A-Za-z]*f[A-Za-z]*[rR]|rm\s+(--recursive\s+--force|--force\s+--recursive|-[A-Za-z]*[rR]\s+--force|--force\s+-[A-Za-z]*[rR]|--recursive\s+-[A-Za-z]*f|-[A-Za-z]*f\s+--recursive)|git\s+push\s+(--force|--force-with-lease|-f)\b|git\s+reset\s+--hard|git\s+clean\s+-[A-Za-z]*f|[Dd][Rr][Oo][Pp]\s+([Tt][Aa][Bb][Ll][Ee]|[Dd][Aa][Tt][Aa][Bb][Aa][Ss][Ee]))' +# `rm -r -f` / `rm -f -r` (and `-r … -f` with intervening flags) are SEPARATED +# short flags that the combined (`-rf`) and long-form alternatives below miss; +# the two `rm\s+-…\s+…-…` alternatives close that gap (F-HOOKS-04). +DESTRUCTIVE='(^|[|;&]|\$\(|`)\s*(env\s+|command\s+)?(rm\s+-[A-Za-z]*[rR][A-Za-z]*f|rm\s+-[A-Za-z]*f[A-Za-z]*[rR]|rm\s+(--recursive\s+--force|--force\s+--recursive|-[A-Za-z]*[rR]\s+--force|--force\s+-[A-Za-z]*[rR]|--recursive\s+-[A-Za-z]*f|-[A-Za-z]*f\s+--recursive)|rm\s+-[A-Za-z]*[rR][A-Za-z]*\s+(-[A-Za-z]+\s+)*-[A-Za-z]*f[A-Za-z]*|rm\s+-[A-Za-z]*f[A-Za-z]*\s+(-[A-Za-z]+\s+)*-[A-Za-z]*[rR][A-Za-z]*|git\s+push\s+(--force|--force-with-lease|-f)\b|git\s+reset\s+--hard|git\s+clean\s+-[A-Za-z]*f|[Dd][Rr][Oo][Pp]\s+([Tt][Aa][Bb][Ll][Ee]|[Dd][Aa][Tt][Aa][Bb][Aa][Ss][Ee]))' # Strip allowed pattern before checking: git reset --hard origin/ CHECKED=$(echo "$COMMAND" | sed -E 's/git +reset +--hard +origin\/[A-Za-z0-9._/-]+//g') @@ -17,7 +44,7 @@ if echo "$CHECKED" | grep -qE "$DESTRUCTIVE"; then fi # --- Indirect destructive patterns (xargs, find -exec) --- -INDIRECT_DESTRUCTIVE='(xargs\s+|find\s+.*-exec\s+)(rm\s+-[A-Za-z]*[rR][A-Za-z]*f|rm\s+-[A-Za-z]*f[A-Za-z]*[rR]|rm\s+(--recursive\s+--force|--force\s+--recursive|-[A-Za-z]*[rR]\s+--force|--force\s+-[A-Za-z]*[rR]|--recursive\s+-[A-Za-z]*f|-[A-Za-z]*f\s+--recursive))|find\s+.*-delete|find\s+.*-exec\s+(bash|sh|zsh|ksh)\s+-c\s+' +INDIRECT_DESTRUCTIVE='(xargs\s+|find\s+.*-exec\s+)(rm\s+-[A-Za-z]*[rR][A-Za-z]*f|rm\s+-[A-Za-z]*f[A-Za-z]*[rR]|rm\s+(--recursive\s+--force|--force\s+--recursive|-[A-Za-z]*[rR]\s+--force|--force\s+-[A-Za-z]*[rR]|--recursive\s+-[A-Za-z]*f|-[A-Za-z]*f\s+--recursive)|rm\s+-[A-Za-z]*[rR][A-Za-z]*\s+(-[A-Za-z]+\s+)*-[A-Za-z]*f[A-Za-z]*|rm\s+-[A-Za-z]*f[A-Za-z]*\s+(-[A-Za-z]+\s+)*-[A-Za-z]*[rR][A-Za-z]*)|find\s+.*-delete|find\s+.*-exec\s+(bash|sh|zsh|ksh)\s+-c\s+' if echo "$COMMAND" | grep -qE "$INDIRECT_DESTRUCTIVE"; then echo "Blocked: indirect destructive command not allowed: $COMMAND" >&2 diff --git a/hooks/pre-edit-safety.sh b/hooks/pre-edit-safety.sh index 8de2968..e3c59d7 100755 --- a/hooks/pre-edit-safety.sh +++ b/hooks/pre-edit-safety.sh @@ -1,11 +1,40 @@ #!/usr/bin/env bash set -euo pipefail + +# jq is a documented hard dependency. When it is missing this guard cannot parse +# the tool payload; rather than dying with a silent `exit 127` (which Claude Code +# treats as a non-blocking error — a fail-OPEN), resolve the behaviour through +# SW_SAFETY_JQ_MISSING_MODE (UX-11): +# on -> fail CLOSED (exit 2) with an explicit message +# metric-only -> (default) log the would-be fail-closed and ALLOW (exit 0) +# off -> silently allow (exit 0) +# A typo collapses to metric-only (the observe-only default). Default is +# metric-only so this hardening does not change the shipped fail-open behaviour +# until an operator opts in with `=on` (post-dogfood promote). +if ! command -v jq >/dev/null 2>&1; then + case "${SW_SAFETY_JQ_MISSING_MODE:-metric-only}" in + on) + echo "[SAFETY-JQ-MISSING] pre-edit-safety: jq not found on PATH — failing closed (exit 2). Install jq (e.g. 'brew install jq') to run this guard." >&2 + exit 2 ;; + off) + exit 0 ;; + metric-only|*) + echo "[SAFETY-JQ-MISSING] metric-only: pre-edit-safety would fail closed (exit 2) — jq not found on PATH; allowing this call. Install jq, or set SW_SAFETY_JQ_MISSING_MODE=on to enforce." >&2 + exit 0 ;; + esac +fi + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INPUT=$(cat) FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty') -# Sensitive file patterns (aligned with pre-bash-safety.sh) -SENSITIVE='(\.(env|key|pem|p12|pfx|jks|keystore)\b|credentials\b|secret\b|id_rsa\b|id_ed25519\b|id_ecdsa\b|\.npmrc$|\.pypirc$)' +# Sensitive file patterns (aligned with pre-bash-safety.sh). F-HOOKS-03: the +# `credentials`/`secret` arms are anchored to secret-bearing EXTENSIONS so +# legitimate source/doc files (credentials.ts, secret.md) are NOT false-blocked, +# while secret-bearing config (credentials.json, app-secret.yaml) and key +# material (.env, *.key, id_rsa) still are. id_rsa* is basename-anchored so +# `id_rsa_test.pub` stays allowed. +SENSITIVE='(\.(env|key|pem|p12|pfx|jks|keystore)\b|\.npmrc$|\.pypirc$|(^|/)(id_rsa|id_ed25519|id_ecdsa)\b|(credentials|secret)[A-Za-z0-9._-]*\.(json|ya?ml|env|ini|cfg|conf|xml|properties|toml)\b)' if echo "$FILE_PATH" | grep -qiE "$SENSITIVE"; then echo "Blocked: editing sensitive file not allowed: $FILE_PATH" >&2 @@ -47,9 +76,21 @@ case "$FILE_PATH" in source "$SCRIPT_DIR/lib/state-authority.sh" OLD_STRING=$(printf '%s' "$INPUT" | jq -r '.tool_input.old_string // empty' 2>/dev/null || true) NEW_STRING=$(printf '%s' "$INPUT" | jq -r '.tool_input.new_string // empty' 2>/dev/null || true) - if state_field_change_blocked "$FILE_PATH" "$OLD_STRING" "$NEW_STRING"; then - jq -nc '{decision:"block", reason:"hook_owned_field_violation"}' - exit 0 + # state_field_change_blocked echoes the violated registry key on stdout + # and returns 0 when a hook-owned field would change. The DENY is gated by + # SW_STATE_FIELD_GUARD_MODE (proposal 4 / ST-03): `on` blocks, `metric-only` + # (default) logs + allows, `off` allows silently. Default is metric-only so + # populating HOOK_OWNED_FIELDS does not change the shipped allow behaviour. + if _SFG_FIELD=$(state_field_change_blocked "$FILE_PATH" "$OLD_STRING" "$NEW_STRING"); then + case "${SW_STATE_FIELD_GUARD_MODE:-metric-only}" in + on) + jq -nc --arg f "$_SFG_FIELD" \ + '{decision:"block", reason:("hook_owned_field_violation: " + $f + " is a hook-owned, append-only field written exclusively by the autopilot Stop / PreCompact / checkpoint hooks. Do not Edit it directly — let the hooks append. See docs/state-schema.md.")}' + exit 0 ;; + off) ;; + metric-only|*) + echo "[STATE-FIELD-GUARD] metric-only: would block hook_owned_field_violation: ${_SFG_FIELD} (Edit to $FILE_PATH); set SW_STATE_FIELD_GUARD_MODE=on to enforce. See docs/state-schema.md." >&2 ;; + esac fi ;; esac diff --git a/hooks/pre-write-safety.sh b/hooks/pre-write-safety.sh index df8eebf..cc46668 100755 --- a/hooks/pre-write-safety.sh +++ b/hooks/pre-write-safety.sh @@ -1,11 +1,40 @@ #!/usr/bin/env bash set -euo pipefail + +# jq is a documented hard dependency. When it is missing this guard cannot parse +# the tool payload; rather than dying with a silent `exit 127` (which Claude Code +# treats as a non-blocking error — a fail-OPEN), resolve the behaviour through +# SW_SAFETY_JQ_MISSING_MODE (UX-11): +# on -> fail CLOSED (exit 2) with an explicit message +# metric-only -> (default) log the would-be fail-closed and ALLOW (exit 0) +# off -> silently allow (exit 0) +# A typo collapses to metric-only (the observe-only default). Default is +# metric-only so this hardening does not change the shipped fail-open behaviour +# until an operator opts in with `=on` (post-dogfood promote). +if ! command -v jq >/dev/null 2>&1; then + case "${SW_SAFETY_JQ_MISSING_MODE:-metric-only}" in + on) + echo "[SAFETY-JQ-MISSING] pre-write-safety: jq not found on PATH — failing closed (exit 2). Install jq (e.g. 'brew install jq') to run this guard." >&2 + exit 2 ;; + off) + exit 0 ;; + metric-only|*) + echo "[SAFETY-JQ-MISSING] metric-only: pre-write-safety would fail closed (exit 2) — jq not found on PATH; allowing this call. Install jq, or set SW_SAFETY_JQ_MISSING_MODE=on to enforce." >&2 + exit 0 ;; + esac +fi + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INPUT=$(cat) FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty') -# Sensitive file patterns (aligned with pre-bash-safety.sh) -SENSITIVE='(\.(env|key|pem|p12|pfx|jks|keystore)\b|credentials\b|secret\b|id_rsa\b|id_ed25519\b|id_ecdsa\b|\.npmrc$|\.pypirc$)' +# Sensitive file patterns (aligned with pre-bash-safety.sh). F-HOOKS-03: the +# `credentials`/`secret` arms are anchored to secret-bearing EXTENSIONS so +# legitimate source/doc files (credentials.ts, secret.md) are NOT false-blocked, +# while secret-bearing config (credentials.json, app-secret.yaml) and key +# material (.env, *.key, id_rsa) still are. id_rsa* is basename-anchored so +# `id_rsa_test.pub` stays allowed. +SENSITIVE='(\.(env|key|pem|p12|pfx|jks|keystore)\b|\.npmrc$|\.pypirc$|(^|/)(id_rsa|id_ed25519|id_ecdsa)\b|(credentials|secret)[A-Za-z0-9._-]*\.(json|ya?ml|env|ini|cfg|conf|xml|properties|toml)\b)' if echo "$FILE_PATH" | grep -qiE "$SENSITIVE"; then echo "Blocked: writing to sensitive file not allowed: $FILE_PATH" >&2 @@ -50,9 +79,21 @@ case "$FILE_PATH" in if [ -f "$FILE_PATH" ]; then OLD_STRING=$(cat "$FILE_PATH" 2>/dev/null || true) fi - if state_field_change_blocked "$FILE_PATH" "$OLD_STRING" "$NEW_STRING"; then - jq -nc '{decision:"block", reason:"hook_owned_field_violation"}' - exit 0 + # state_field_change_blocked echoes the violated registry key on stdout + # and returns 0 when a hook-owned field would change. The DENY is gated by + # SW_STATE_FIELD_GUARD_MODE (proposal 4 / ST-03): `on` blocks, `metric-only` + # (default) logs + allows, `off` allows silently. Default is metric-only so + # populating HOOK_OWNED_FIELDS does not change the shipped allow behaviour. + if _SFG_FIELD=$(state_field_change_blocked "$FILE_PATH" "$OLD_STRING" "$NEW_STRING"); then + case "${SW_STATE_FIELD_GUARD_MODE:-metric-only}" in + on) + jq -nc --arg f "$_SFG_FIELD" \ + '{decision:"block", reason:("hook_owned_field_violation: " + $f + " is a hook-owned, append-only field written exclusively by the autopilot Stop / PreCompact / checkpoint hooks. Do not Write it directly — let the hooks append. See docs/state-schema.md.")}' + exit 0 ;; + off) ;; + metric-only|*) + echo "[STATE-FIELD-GUARD] metric-only: would block hook_owned_field_violation: ${_SFG_FIELD} (Write to $FILE_PATH); set SW_STATE_FIELD_GUARD_MODE=on to enforce. See docs/state-schema.md." >&2 ;; + esac fi ;; esac diff --git a/skills/autopilot/references/state-file.md b/skills/autopilot/references/state-file.md index 32372e9..4f8f0a7 100644 --- a/skills/autopilot/references/state-file.md +++ b/skills/autopilot/references/state-file.md @@ -24,7 +24,10 @@ tickets: - logical_id: {parent-slug}-part-{N} # one entry per split-plan ticket, in topological order ticket_dir: {ticket-dir from split-plan} status: pending - steps: {scout: pending, impl: pending, ship: pending} + steps: # canonical FLAT/block form — `steps.` is a STRING on its own line + scout: pending + impl: pending + ship: pending invocation_method: {scout: unknown, impl: unknown, ship: unknown} runtime_metrics: [] # append-only, written by Stop / PreCompact hooks # Sample entry (one full session_end snapshot, all 7 canonical keys): @@ -196,12 +199,17 @@ half-finished run that was moved by mistake. ## `runtime_metrics:` schema `runtime_metrics:` is an **append-only** list written exclusively by the -Stop hook (`hooks/autopilot-continue.sh`) and the PreCompact hook -(`hooks/pre-compact-save.sh`). The list survives ticket completion -(Split State File Cleanup keeps it intact when moving the state file to -`briefs/done/`). Skills MUST NOT write `runtime_metrics:` directly — -hook-only ownership keeps the schema observable from a single audit -point. +following six hooks (the canonical writer set is the `Sourced by:` header of +`hooks/lib/runtime-metrics.sh`, kept in sync by a contract test): +`hooks/autopilot-continue.sh` (session_end), `hooks/pre-compact-save.sh` +(session_compaction), `hooks/impl-checkpoint-guard.sh` (session_end), +`hooks/scout-checkpoint-guard.sh` (session_end), +`hooks/pre-next-scout-auto-compact.sh` (auto_compact_inject), and +`hooks/post-ship-state-auto-compact.sh` (auto_compact_inject). The list +survives ticket completion (Split State File Cleanup keeps it intact when +moving the state file to `briefs/done/`). Skills MUST NOT write +`runtime_metrics:` directly — hook-only ownership keeps the schema observable +from a single audit point. Value domains for `boundary` and `stop_reason`, plus the Stop hook's discrimination heuristic, are defined in diff --git a/skills/brief/SKILL.md b/skills/brief/SKILL.md index 677bfaa..d1ceb84 100644 --- a/skills/brief/SKILL.md +++ b/skills/brief/SKILL.md @@ -54,7 +54,7 @@ User input: $ARGUMENTS Parse `$ARGUMENTS`: - **Preferred new key — `chain=`**: extract `chain=` if present. **Value normalization**: trim whitespace and lowercase the value (`chain=ON`, `chain=Off`, `chain= on ` normalize to `on`/`off`). Token `chain=` is matched case-insensitively. Accepted: `on` (default if neither `chain=` nor `mode=` is supplied), `off`. Any other value → stop and emit `ERROR: invalid chain=. Use chain=on or chain=off` (substituting the offending value); do NOT create the brief directory, do NOT write `brief.md`, do NOT write `autopilot-policy.yaml`, and do NOT write `auto-kick.yaml`; exit non-zero. Mapping for downstream / legacy reasoning: `chain=on` ≡ `mode=auto`, `chain=off` ≡ `mode=manual`. -- **Deprecated alias — `mode=`**: extract `mode=` if present. **Value normalization**: trim whitespace and lowercase the value (`mode=AUTO`, `mode=Manual`, `mode= auto ` normalize to `auto`/`manual`). Token `mode=` is matched case-insensitively. Accepted: `auto` (treated as `chain=on`), `manual` (treated as `chain=off`). Any other value → stop and emit the invalid-mode error (see ## Error Handling for exact message + side-effect contract). **When `mode=` is supplied, also emit to stderr the single line `WARNING: 'mode=' is deprecated and will be removed in vX.(Y+1).0. Use 'chain=on' instead of 'mode=auto', 'chain=off' instead of 'mode=manual'.`** (verbatim literal, including the leading `WARNING:` and the surrounding single quotes). The warning is informational; processing continues with the `mode=` value mapped to the equivalent `chain=` value. +- **Deprecated alias — `mode=`**: extract `mode=` if present. **Value normalization**: trim whitespace and lowercase the value (`mode=AUTO`, `mode=Manual`, `mode= auto ` normalize to `auto`/`manual`). Token `mode=` is matched case-insensitively. Accepted: `auto` (treated as `chain=on`), `manual` (treated as `chain=off`). Any other value → stop and emit the invalid-mode error (see ## Error Handling for exact message + side-effect contract). **When `mode=` is supplied, also emit to stderr the single line `WARNING: 'mode=' is deprecated and will be removed in v9.0.0. Use 'chain=on' instead of 'mode=auto', 'chain=off' instead of 'mode=manual'.`** (verbatim literal, including the leading `WARNING:` and the surrounding single quotes). The warning is informational; processing continues with the `mode=` value mapped to the equivalent `chain=` value. - **Simultaneous specification — `chain=` and `mode=` both present**: stop and emit `ERROR: 'chain=' and 'mode=' cannot be combined. Use 'chain=' (preferred).` Do NOT silent-rewrite; do NOT pick one and ignore the other. Do NOT create the brief directory, do NOT write `brief.md`, do NOT write `autopilot-policy.yaml`, and do NOT write `auto-kick.yaml`. Exit non-zero. (This mirrors the v6.0.0 `auto=true` defensive stance — no silent rewrites of ambiguous argument intent.) - **Default when both keys are omitted**: `chain=on` (equivalent to legacy `mode=auto`). The default preserves the prior `mode=auto`-default behavior so existing user-typed `/brief ""` invocations continue to chain into `/create-ticket` and `/autopilot`. - **`auto=true` removal (v6.0.0)**: if `auto=true` (case-insensitive) appears in `$ARGUMENTS`, stop and emit the v6.0.0 removal error (see ## Error Handling). The removal is intentional and `auto=true` is NOT silently rewritten. @@ -78,7 +78,7 @@ Parse `$ARGUMENTS`: Conduct an iterative Q&A to gather comprehensive requirements. -**mode independence guard (load-bearing — preserved during the `chain=` / `mode=` deprecation period; defensive prose now covers BOTH the new `chain` argument AND the legacy `mode` alias)**: Phase 2 Structured Interview (Socratic) **MUST** run regardless of the parsed `mode` value (`auto` or `manual`, including when `mode=` is omitted and defaults to `auto`). The same independence rule applies to the new canonical `chain` argument introduced in vX.Y.0 — Phase 2 MUST run regardless of the parsed `chain` value (`on` or `off`, including when `chain=` is omitted and defaults to `on` ≡ legacy `mode=auto`). The `mode` argument **MUST NOT** be interpreted as a signal to skip, shorten, or bypass Phase 2 — it has **no effect whatsoever** on Phase 2's execution. The same MUST-NOT applies to the new `chain` argument: it is independence-protected from Phase 2 alongside `mode`, and has no effect whatsoever on Phase 2's execution. Any non-interactive wording elsewhere in this skill (e.g. the Finalization Phase Step 2 chain-confirmation context gated on `mode=auto` ≡ `chain=on`) is scoped to that specific step and **MUST NOT** be generalized to Phase 2. The **ONLY** condition under which Phase 2 is skipped is the existing **"Non-interactive environment fallback"** below (triggered strictly by `AskUserQuestion` itself being unavailable or returning an error, e.g. `claude -p` / CI automation without a TTY) — **not** by the value of `mode` or `chain`. This guard is scheduled for removal in vX.(Y+1).0 alongside the `mode=` alias: `chain={on,off}` does not lexically suggest interview-policy, so the defensive prose becomes unnecessary once the alias is gone and Phase 2 independence can be left implicit. +**mode independence guard (load-bearing — preserved during the `chain=` / `mode=` deprecation period; defensive prose now covers BOTH the new `chain` argument AND the legacy `mode` alias)**: Phase 2 Structured Interview (Socratic) **MUST** run regardless of the parsed `mode` value (`auto` or `manual`, including when `mode=` is omitted and defaults to `auto`). The same independence rule applies to the new canonical `chain` argument introduced in vX.Y.0 — Phase 2 MUST run regardless of the parsed `chain` value (`on` or `off`, including when `chain=` is omitted and defaults to `on` ≡ legacy `mode=auto`). The `mode` argument **MUST NOT** be interpreted as a signal to skip, shorten, or bypass Phase 2 — it has **no effect whatsoever** on Phase 2's execution. The same MUST-NOT applies to the new `chain` argument: it is independence-protected from Phase 2 alongside `mode`, and has no effect whatsoever on Phase 2's execution. Any non-interactive wording elsewhere in this skill (e.g. the Finalization Phase Step 2 chain-confirmation context gated on `mode=auto` ≡ `chain=on`) is scoped to that specific step and **MUST NOT** be generalized to Phase 2. The **ONLY** condition under which Phase 2 is skipped is the existing **"Non-interactive environment fallback"** below (triggered strictly by `AskUserQuestion` itself being unavailable or returning an error, e.g. `claude -p` / CI automation without a TTY) — **not** by the value of `mode` or `chain`. This guard is scheduled for removal in v9.0.0 alongside the `mode=` alias: `chain={on,off}` does not lexically suggest interview-policy, so the defensive prose becomes unnecessary once the alias is gone and Phase 2 independence can be left implicit. **Caps (load-bearing for contract)**: - At most **3 questions per round** (single `AskUserQuestion` call holds up to 3 items). @@ -152,7 +152,7 @@ interview_complete: {true|false} ``` - `chain` (canonical, vX.Y.0+) is the literal scalar (`on` or `off`) derived from the parsed argument in the Argument Parsing step (`chain=on|off` if supplied; otherwise mapped from the deprecated `mode=auto|manual` alias as `auto → on`, `manual → off`; otherwise the default `on`). Sample frontmatter MUST include this `chain: (on|off)` line. -- `mode` (legacy, retained for one minor during the deprecation period — slated for removal in vX.(Y+1).0) is the literal scalar (`auto` or `manual`) parsed from `$ARGUMENTS` in the Argument Parsing step (defaulting to `auto` when both `chain=` and `mode=` are omitted; equivalent to the derived `chain` value via `on ↔ auto` / `off ↔ manual`). During this deprecation window both keys are written to every new brief so legacy `/create-ticket` consumers that have not yet adopted the `chain:` reader keep working unchanged. It is REQUIRED in v6.0.0+ (carried forward as `mode: {auto|manual}` per the deprecation alias contract). Super-legacy briefs written before v6.0.0 may lack this key entirely; downstream readers (notably `/create-ticket brief=`) follow the precedence rule **`chain:` precedes `mode:`** — `chain:` is read first if present, otherwise `mode:` is read for backward compatibility, otherwise the reader defaults to `chain=on` (≡ `mode: auto`). +- `mode` (legacy, retained through the deprecation period — slated for removal in v9.0.0, the next major) is the literal scalar (`auto` or `manual`) parsed from `$ARGUMENTS` in the Argument Parsing step (defaulting to `auto` when both `chain=` and `mode=` are omitted; equivalent to the derived `chain` value via `on ↔ auto` / `off ↔ manual`). During this deprecation window both keys are written to every new brief so legacy `/create-ticket` consumers that have not yet adopted the `chain:` reader keep working unchanged. It is REQUIRED in v6.0.0+ (carried forward as `mode: {auto|manual}` per the deprecation alias contract). Super-legacy briefs written before v6.0.0 may lack this key entirely; downstream readers (notably `/create-ticket brief=`) follow the precedence rule **`chain:` precedes `mode:`** — `chain:` is read first if present, otherwise `mode:` is read for backward compatibility, otherwise the reader defaults to `chain=on` (≡ `mode: auto`). - `interview_complete` is the literal scalar recorded at the end of Phase 2 (`true` if at least one round ran to a user response; `false` if Phase 2 was skipped via the non-interactive fallback or no round produced a response). - **Do NOT emit `split:`** (obsolete — decomposition is `/create-ticket`'s job). - **Do NOT emit `ticket_count:`** (obsolete — decomposition is `/create-ticket`'s job). @@ -163,7 +163,7 @@ interview_complete: {true|false} 1. Read `.simple-workflow/kb/index.yaml`, filter the `autopilot` section (historical decision patterns produced by `/tune` analysis), and apply confidence-based 3-tier judgment per gate (`>= 0.7` → annotate with `# kb-suggested`; `0.5-0.7` → `# [low confidence]`; `< 0.5` → conservative default). Apply size-scoped pattern priority before falling back to `scope=general`. Full integration logic: see [kb-policy-integration](references/kb-policy-integration.md). 2. Determine default policy values based on the user's risk tolerance answers from Phase 2 (maps to conservative/moderate/aggressive). If Phase 2 was skipped (`interview_complete: false`), default to **conservative** and emit default gates. -3. Write to `.simple-workflow/backlog/briefs/active/{slug}/autopilot-policy.yaml` regardless of Phase 2 outcome **and regardless of the parsed `chain` value** (or the legacy `mode` value, while the alias is accepted; the top-level `gates:` line MUST be present). Even in `chain=off` (legacy `mode=manual`), the brief-level `autopilot-policy.yaml` is written as a **rescue path**: per-ticket propagation is suppressed by `/create-ticket` (so manual-mode tickets work like bare-mode tickets via `/impl`'s FIFO selector), but the brief-level policy file remains so a user can later opt into autopilot by running `/autopilot {slug}` directly — the autopilot's brief-level policy fallback (see `skills/autopilot/SKILL.md` Phase 1 step 2, "Brief optionality") consumes this file. +3. Write to `.simple-workflow/backlog/briefs/active/{slug}/autopilot-policy.yaml` regardless of Phase 2 outcome **and regardless of the parsed `chain` value** (or the legacy `mode` value, while the alias is accepted; the top-level `gates:` line MUST be present). Even in `chain=off` (legacy `mode=manual`), the brief-level `autopilot-policy.yaml` is written as a **rescue path**: per-ticket propagation is suppressed by `/create-ticket` (so manual-mode tickets work like bare-mode tickets via `/impl`'s FIFO selector), but the brief-level policy file remains so a user can later opt into autopilot by **first re-running `/create-ticket` with the brief set to `chain: on`** (which propagates `autopilot-policy.yaml` to each ticket dir) and then running `/autopilot {slug}`, exactly as Step 3 instructs. Running `/autopilot {slug}` directly on a manual (`chain: off`) brief stops with a re-propagation directive — the autopilot brief-level policy fallback was retired (`skills/autopilot/SKILL.md` Phase 1). The emitted YAML follows the template in [policy-template](references/policy-template.md) (version 1 / risk_tolerance / gates / constraints, with conservative/moderate/aggressive branches inlined as comments). @@ -208,7 +208,7 @@ f. Otherwise (`/create-ticket` succeeded), proceed to the final invocation: ### Step 3 — `chain=off` (≡ legacy `mode=manual`; no chained handoff) -> Legacy-alias heading kept inline for one-minor backward search compatibility: this section is also reachable as **Step 3 — `mode=manual`** while the `mode=` alias remains accepted (slated for removal in vX.(Y+1).0 alongside the alias). +> Legacy-alias heading kept inline for backward search compatibility: this section is also reachable as **Step 3 — `mode=manual`** while the `mode=` alias remains accepted (slated for removal in v9.0.0 alongside the alias). @@ -259,7 +259,7 @@ In all three shapes (`chain=on` success ≡ legacy `mode=auto` success, `chain=o - **`auto=true` argument (v6.0.0 removal)**: Print `ERROR: 'auto=true' has been removed in v6.0.0; use 'mode=auto' or 'mode=manual'` (legacy literal preserved verbatim; the equivalent vX.Y.0+ form is `chain=on` or `chain=off`) and stop. Do NOT create the brief directory, do NOT write `brief.md`, do NOT write `autopilot-policy.yaml`, and do NOT write `auto-kick.yaml`. Exit non-zero. - **Invalid `chain=` (vX.Y.0+ canonical form)**: Print `ERROR: invalid chain=. Use chain=on or chain=off` (substituting the offending value) and stop. Do NOT create the brief directory, do NOT write `brief.md`, do NOT write `autopilot-policy.yaml`, and do NOT write `auto-kick.yaml`. Exit non-zero. - **Invalid `mode=` (legacy alias path)**: Print `ERROR: invalid mode=. Use mode=auto or mode=manual` (substituting the offending value) and stop. Do NOT create the brief directory, do NOT write `brief.md`, do NOT write `autopilot-policy.yaml`, and do NOT write `auto-kick.yaml`. Exit non-zero. (The deprecation warning for `mode=` itself is emitted only when the alias is *valid*; invalid `mode=` values short-circuit to this error without the warning.) -- **Deprecated `mode=` alias supplied (warning, not an error)**: When `mode=` is parsed with a valid value (`auto` or `manual`) and `chain=` is NOT also supplied, emit to stderr the single line `WARNING: 'mode=' is deprecated and will be removed in vX.(Y+1).0. Use 'chain=on' instead of 'mode=auto', 'chain=off' instead of 'mode=manual'.` (verbatim literal). Processing continues — the alias is mapped to the equivalent `chain=` value and the brief is written normally. +- **Deprecated `mode=` alias supplied (warning, not an error)**: When `mode=` is parsed with a valid value (`auto` or `manual`) and `chain=` is NOT also supplied, emit to stderr the single line `WARNING: 'mode=' is deprecated and will be removed in v9.0.0. Use 'chain=on' instead of 'mode=auto', 'chain=off' instead of 'mode=manual'.` (verbatim literal). Processing continues — the alias is mapped to the equivalent `chain=` value and the brief is written normally. - **Simultaneous `chain=` and `mode=` specification (ERROR, exit non-zero)**: Print `ERROR: 'chain=' and 'mode=' cannot be combined. Use 'chain=' (preferred).` and stop. Do NOT silent-rewrite to either key. Do NOT create the brief directory, do NOT write `brief.md`, do NOT write `autopilot-policy.yaml`, and do NOT write `auto-kick.yaml`. Exit non-zero. - **Researcher failure**: Report error. Continue to Phase 2 without investigation summary. - **AskUserQuestion failure in Phase 2**: Skip Phase 2, proceed with researcher findings only; set `interview_complete: false`. diff --git a/skills/brief/references/policy-template.md b/skills/brief/references/policy-template.md index 0fbda1e..d3772b6 100644 --- a/skills/brief/references/policy-template.md +++ b/skills/brief/references/policy-template.md @@ -39,4 +39,5 @@ constraints: eval_panel: auto # auto (default, failure-class eval panel M8/v8.4.0+) grades through a fixed failure-class lens set (L-CORRECTNESS + >=1 more at standard, 5-lens fan-out at exhaustive) instead of a single all-purpose pass and emits a per-ticket [EVAL-PANEL] lenses= mode= observability line after every AC is verdicted; auto activates for a ticket touching >=2 source units or carrying >=1 behavioral AC, on forces it, off disables it (drops the panel + the [EVAL-PANEL] line; byte-for-byte revert to the pre-panel single-pass behaviour). The per-AC [ORACLE-AUDIT] line is unconditional and NOT gated by this switch. See skills/create-ticket/references/autopilot-policy-reference.md constraints.eval_panel and skills/impl/SKILL.md Step 15 selfdoc_verification: auto # auto (default, EC-SELFDOC doc-truthfulness v8.4.0+) activates the EC-SELFDOC evidence channel + the doc-verifier agent: a behavioral AC may compare the unit's OWN declared contract (docstring/declared invariant/type annotation/--help line/README worked-example/advertised size-or-range boundary) against observed runtime behaviour — RUN each advertised example and diff (byte-for-byte if deterministic, explicit tolerance otherwise), feed each advertised boundary a FORBIDDEN value (must reject) + an ALLOWED value (must accept), under the .simple-workflow/scratch/ exec carve-out, fail-open with a Caveat where the build can't be exercised; this is the standing channel for Gate 9 rows R3/R4. off stands the channel + doc-verifier down (other behavioral channels unaffected; Gate 9 R3/R4 fall back to pre-v8.4.0 prose) — byte-for-byte revert. Independent of oracle_verification/independent_evidence/failure_class_coverage/eval_panel. See skills/impl/references/evidence-channels.md, skills/create-ticket/references/ac-quality-criteria.md Gate 9 R3/R4, and skills/create-ticket/references/autopilot-policy-reference.md constraints.selfdoc_verification refute_merge: auto # auto (default, refute-then-synthesize merge v8.4.0+) makes the exhaustive 3-verifier merge refute-driven: a non-critical FAIL raised by any one valid verifier survives as FAIL unless every other valid verifier refutes it (independently rendered PASS/PASS-WITH-CAVEATS on the same AC; silence is not refutation), so a lone reproducing non-critical FAIL is no longer demoted to PASS; off reverts byte-for-byte to the prior majority-merge (a non-critical FAIL needs >=2 verifiers; lone FAIL demoted to PASS). CRITICAL-not-voted-away, the valid<2 quorum, and the severity ladder are mode-invariant; the merge runs only in the exhaustive multi-verifier branch. See skills/create-ticket/references/autopilot-policy-reference.md constraints.refute_merge and skills/impl/SKILL.md Step 15 + accept_set_conformance: auto # auto (default, Advertised-Accept-Set Conformance v8.5.0+) makes the ac-evaluator EXECUTE a generative grammar-complement sweep (black-box vs an independent spec oracle, MR-FINITE/MR-ALPHABET/MR-CANONICAL/MR-KEYFAITH) for any AC whose boundary advertises strict/canonical/lossless/limit or shares an input class with a sibling, and the hooks/accept-set-verify.sh PostToolUse gate deterministically verifies the persisted ## Accept-set sweep line (a triggered boundary not run / a shallow astral sweep / a sliced corpus / an authoritative divergence not driven to FAIL is flagged); off skips the EXECUTED sweep, reverting byte-for-byte to read-only strictness reasoning + a one-line Caveat. Independent of verification_depth/oracle_verification/eval_panel. See skills/create-ticket/references/autopilot-policy-reference.md constraints.accept_set_conformance and skills/impl/SKILL.md Step 15 ``` diff --git a/skills/create-ticket/SKILL.md b/skills/create-ticket/SKILL.md index 27357df..0d89aa8 100644 --- a/skills/create-ticket/SKILL.md +++ b/skills/create-ticket/SKILL.md @@ -91,7 +91,7 @@ Per-mode deltas: `MUST invoke the simple-workflow:decomposer via the Agent tool` with an inline `scope_context` spawn prompt per [references/spec-decomposer-input.md](references/spec-decomposer-input.md) Form B. Body: `Input form: scope_context`; `Parent slug: {parent-slug}` (kebab-case of description); `## Context` = description verbatim; `## Investigation Summary` = Phase 1 transient `investigation.md`; `## Socratic Answers` (if any) = one bullet per answer. -Receive `## Result` (Status / Parent slug / Tickets (each ticket also carrying a `peer_set: true|false` Gate 10 hint, and a `shared_conventions:` line when `peer_set` is true) / Topological order / Rationale). Failures: `Status: failed` → `ERROR: decomposer failed — `; unavailable → `ERROR: decomposer agent unavailable`; empty Tickets → `ERROR: decomposer returned zero tickets`; cycle → `ERROR: circular dependency detected among tickets: `. All exit non-zero, atomic. +Receive `## Result` (Status / Parent slug / Tickets (each ticket also carrying a `peer_set: true|false` Gate 10 hint, and a `shared_conventions:` line when `peer_set` is true) / Topological order / Rationale / a mandatory-present cross-ticket `shared_input_boundary:` signal naming each external input class consumed by `>=2` tickets and the sibling ticket ids, or `none`). Forward each sibling ticket's `shared_input_boundary` membership (the shared input class + the sibling ticket ids) into BOTH that ticket's planner spawn AND that ticket's ticket-evaluator spawn (the Gate 7 sibling-guard / Gate 9 upstream input-axis signal), alongside its per-ticket `peer_set:` / `shared_conventions:` hint, so the grader can see the membership the decomposer observed across ticket boundaries. Failures: `Status: failed` → `ERROR: decomposer failed — `; unavailable → `ERROR: decomposer agent unavailable`; empty Tickets → `ERROR: decomposer returned zero tickets`; cycle → `ERROR: circular dependency detected among tickets: `. All exit non-zero, atomic. ### Step B-5: Synthesize `scope_context` and invoke `decomposer` (Brief Mode) diff --git a/skills/create-ticket/references/ac-quality-criteria.md b/skills/create-ticket/references/ac-quality-criteria.md index 13197e1..8b26145 100644 --- a/skills/create-ticket/references/ac-quality-criteria.md +++ b/skills/create-ticket/references/ac-quality-criteria.md @@ -186,7 +186,7 @@ A probe entry that appears in none of the three buckets is a Gate 6.5 FAIL — t **Scope**: per-AC, with a ticket-wide kill switch (see below). -**Motivation**: a test whose expected value is produced by the code under test — or by re-applying the implementation's own rounding / formatting — is self-confirming: it passes whenever the code is internally consistent, even when the code is wrong. This is the defect class that shipped a WCAG contrast solver accepting on a 2-decimal **rounded** ratio (falsely reporting a target as met) past a green 93-test suite, because every test re-measured with the same rounded value the code produced. Gate 7 makes that circularity an authoring-time FAIL. +**Motivation**: a test whose expected value is re-derived from the implementation's own rounded / formatted output is self-confirming — it passes whenever the code is internally consistent, even when wrong. This is the defect class where a test re-thresholds the same rounded value the code produced (falsely reporting a numeric target as met) past a fully green test suite, because every test re-measured with that rounded value. Gate 7 makes that circularity an authoring-time FAIL. **Definition**: an AC is **computational** when its PASS/FAIL hinges on a COMPUTED numeric or algorithmic value — a value the implementation calculates rather than a structural fact. Non-exhaustive cues: a contrast / luminance / color-space ratio, a rounding or precision threshold, a hash / checksum / collision rate, a financial or unit conversion, a parser / serializer round-trip, a distance / similarity / statistical metric, any "within X of Y" or "≥ / ≤ a numeric target" outcome. An AC that is purely structural (file-grep / counter / exit-code verifiable) is **not** computational and is graded `n/a` for Gate 7. @@ -200,11 +200,11 @@ A probe entry that appears in none of the three buckets is a Gate 6.5 FAIL — t **Binding rule** (per computational AC): each computational AC MUST either (a) name an independent oracle + a raw-value tolerance, OR (b) declare the no-oracle fallback above, OR (c) be rewritten as a static AC. A computational AC with none of these is a Gate 7 FAIL. -**Adversarial-input requirement** (every **external-input boundary (computational or behavioral)** — broadened in M3, v8.4.0+): additionally, when an AC's value OR observable behaviour comes from a function that takes external / untrusted input — whether the AC is computational (a computed value) OR behavioral (an observable runtime outcome: a returned value, status code, thrown error, wire payload) — the AC MUST require adversarial / non-finite / out-of-range coverage (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range / out-of-gamut) — independently of the oracle vs no-oracle path. (The oracle + raw-value + tolerance requirements above stay computational-only; only this hostile-input coverage requirement broadens to behavioral external-input ACs, because a DoS hang or a contract-violating error path is reachable on bad input regardless of whether the output is a computed number.) This is what catches DoS hangs and contract-violating outputs on *bad* input, not merely wrong values on *good* input (the motivating dogfood build also shipped a non-finite-input DoS hang and an out-of-range channel leak, both invisible to fixed in-gamut fixtures). A computational OR behavioral AC on an externally-fed function with zero adversarial coverage is a Gate 7 FAIL. The adversarial coverage MUST include at least one **parse-accepted-then-overflows** vector — a value that passes syntactic parsing but yields a non-finite / out-of-range intermediate (e.g. `oklch(0.5 1e400 30)` → Infinity chroma) — NOT only parse-rejected `NaN` / `Infinity` keyword tokens. The parser rejects the latter cheaply at the door, so they exercise the wrong path; the real DoS / corrupt-success bugs live in the values the parser ACCEPTS. (A v8.2.0 dogfood shipped exactly this: the generated test used `oklch(NaN ..)` / `oklch(Infinity ..)`, rejected in ~0 ms, while `oklch(0.5 1e400 30)` parsed and hung an unbounded clamp loop.) +**Adversarial-input requirement** (every **external-input boundary (computational or behavioral)** — broadened in M3, v8.4.0+): additionally, when an AC's value OR observable behaviour comes from a function that takes external / untrusted input — whether the AC is computational (a computed value) OR behavioral (an observable runtime outcome: a returned value, status code, thrown error, wire payload) — the AC MUST require adversarial / non-finite / out-of-range coverage (`NaN`, `Infinity`, empty, malformed, oversized, out-of-range) — independently of the oracle vs no-oracle path. (The oracle + raw-value + tolerance requirements above stay computational-only; only this hostile-input coverage requirement broadens to behavioral external-input ACs, because a DoS hang or a contract-violating error path is reachable on bad input regardless of whether the output is a computed number.) This is what catches DoS hangs and contract-violating outputs on *bad* input, not merely wrong values on *good* input (the motivating dogfood build also shipped a non-finite-input DoS hang and an out-of-range channel leak, both invisible to fixed valid-only fixtures). A computational OR behavioral AC on an externally-fed function with zero adversarial coverage is a Gate 7 FAIL. The adversarial coverage MUST include at least one **parse-accepted-then-overflows** vector — an input the parser ACCEPTS (satisfies the grammar) that yields a non-finite / out-of-range intermediate deeper in the algorithm (e.g. a syntactically-valid magnitude that overflows after a conversion) — NOT only parse-rejected `NaN` / `Infinity` keyword tokens. The parser rejects the latter cheaply at the door, so they exercise the wrong path; the real DoS / corrupt-success bugs live in the values the parser ACCEPTS. (A real dogfood shipped exactly this: the generated test used parse-rejected `NaN` / `Infinity` keyword tokens, rejected in ~0 ms, while a parse-accepted overflowing magnitude got through and hung an unbounded clamp loop.) -**Shared / sibling-guard requirement** (computational ACs sharing an input boundary): when a computational AC's function shares an input parser / validation boundary with sibling tools (e.g. several MCP tools that each parse the same CSS-color string), the input-validation guard (finiteness, range, gamut) MUST either (a) live in the SHARED boundary so every sibling inherits it, OR (b) be replicated AND adversarially tested in EVERY sibling tool that accepts that input class. A guard wired into one consumer but absent from its analogous siblings is a Gate 7 FAIL — this is the `## Modifications` sibling-artifact rule (`CLAUDE.md`) enforced at the AC level. (Dogfood evidence: a finite-components guard added to the solver but not to the analogous `gamut_map` / `parse_color` tools left a live DoS hang reachable through the unguarded siblings.) +**Shared / sibling-guard requirement** (computational ACs sharing an input boundary): when a computational AC's function shares an input parser / validation boundary with sibling tools (e.g. several tools / endpoints that each parse the same input class), the input-validation guard (finiteness, range) MUST either (a) live in the SHARED boundary so every sibling inherits it, OR (b) be replicated AND adversarially tested in EVERY sibling tool that accepts that input class. A guard wired into one consumer but absent from its analogous siblings is a Gate 7 FAIL — this is the `## Modifications` sibling-artifact rule (`CLAUDE.md`) enforced at the AC level. (Dogfood evidence: a validation guard added to one consumer but absent from its analogous siblings leaves a live failure reachable through the unguarded siblings.) **The sibling set is not confined to one ticket — it spans the whole product across ticket boundaries.** When separate tickets EACH create a unit consuming one common input class (so each ticket creates a single unit and its per-ticket `peer_set` is `false`), those units still form one sibling set that the per-ticket peer signal cannot see; the `decomposer` surfaces it as the cross-ticket `shared_input_boundary` signal forwarded into each sibling's planner and ticket-evaluator spawn (see `agents/decomposer.md`). This is the SAME input-validation-strictness axis Gate 9 R2 already owns, reached across tickets — NOT a new gate and NOT a re-home into Gate 10. **A sibling that DELEGATES to the shared parser is NOT automatically n/a.** A delegating sibling still re-exposes the shared boundary through its own surface (a round-trip / re-serialization / re-validation path that can leak on an input the shared parser accepted), so it MUST carry its own hostile / accept-set row (the Gate 9 R2 strictness-leniency probe, boundary-keyed) OR a one-line JUSTIFIED `n/a` a reviewer can challenge — delegation is NOT an automatic n/a, never a silent skip, and 'delegates to the shared parser' is itself NOT an acceptable n/a reason (the row is owed precisely because delegation re-exposes the boundary; an acceptable n/a names a different reason, e.g. the sibling forwards an opaque handle and never re-serializes the parsed value). (Dogfood evidence: a delegating sibling that merely round-tripped a value the shared parser had accepted still raised an unguarded error on a valid-but-extreme magnitude — the leak the shared parser's guard did not cover.) -**Algorithm-vs-algorithm differential (depth-gated, EC-DIFFERENTIAL)**: when a computational AC's contract admits a **second, independent ALGORITHM** for the same result (e.g. gamut mapping by CSS-MINDE vs chroma-clamping; two independent sorts; two serializers), at the `thorough` / `exhaustive` evidence_floor the verification SHOULD cross-check the implementation against that second algorithm **within an explicit tolerance**, not merely assert the output satisfies a membership / invariant test. A membership check (`inGamut`, "result is sorted") is **necessary-not-sufficient** — a wrong-but-in-range result passes it. Where no second independent algorithm exists for the contract, the membership / property coverage stands and the AC records a one-line Caveat — this is never a force-FAIL. +**Algorithm-vs-algorithm differential (depth-gated, EC-DIFFERENTIAL)**: when a computational AC's contract admits a **second, independent ALGORITHM** for the same result (e.g. two independent sorts; two serializers), at the `thorough` / `exhaustive` evidence_floor the verification SHOULD cross-check the implementation against that second algorithm **within an explicit tolerance**, not merely assert the output satisfies a membership / invariant test. A membership check ("result is sorted", "result is in-range") is **necessary-not-sufficient** — a wrong-but-in-range result passes it. Where no second independent algorithm exists for the contract, the membership / property coverage stands and the AC records a one-line Caveat — this is never a force-FAIL. **Kill switch**: when `{ticket-dir}/autopilot-policy.yaml` sets `constraints.oracle_verification: off`, Gate 7 is graded `n/a` ticket-wide (restores pre-v8.2.0 authoring behaviour). Absent field / absent policy / unknown value → `auto` (Gate 7 active — fail-safe). @@ -237,12 +237,12 @@ A probe entry that appears in none of the three buckets is a Gate 6.5 FAIL — t **Motivation**: Gates 1-8 GRADE the ACs that were written; none of them GENERATE the ACs that should exist. AC derivation is feature-driven — the planner writes ACs for the behaviour the feature adds, so a whole failure class (a full-domain invariant violated only at an extreme, a DoS hang on hostile input, a docstring that lies about behaviour, an advertised example that no longer builds) is silently uncovered because no AC ever named it. Gate 9 makes AC derivation **coverage-driven**: for each external boundary the Scope touches, it ENUMERATES the four failure-class rows that MUST each yield >=1 AC or carry an explicit `n/a` justification. Gate 9 only ENUMERATES which boundaries need an AC; Gate 7 and Gate 8 GRADE that AC once it exists (Gate 9 does not re-grade evidence independence). -**Boundary key**: a **Scope-touched external boundary** is any externally reachable surface the ticket's `### Scope` adds or changes — a public/exported function, a CLI subcommand, an HTTP/RPC endpoint, an exported API symbol, a file-format or wire-format, or a parser. Gate 9 keys ONLY on these AND on `>=2`-peer sets (a family of analogous sibling tools sharing an input class). It does **not** fire on internal helpers, private functions, or single-call-site refactors (routine-ticket flood prevention) — a ticket that touches no external boundary is graded `n/a` for Gate 9 in full. +**Boundary key**: a **Scope-touched external boundary** is any externally reachable surface the ticket's `### Scope` adds or changes — a public/exported function, a CLI subcommand, an HTTP/RPC endpoint, an exported API symbol, a file-format or wire-format, or a parser. Gate 9 keys ONLY on these AND on `>=2`-peer sets (a family of analogous sibling tools sharing an input class) — and that family is not confined to one ticket: when separate tickets EACH create a unit consuming one common external input class, every such unit is a Scope-touched boundary for Gate 9 even when its own ticket creates a single unit (per-ticket `peer_set: false`), and the cross-ticket `shared_input_boundary` signal the `decomposer` forwards (see Gate 7 sibling-guard) names which tickets belong; a member that DELEGATES the input handling to a shared parser created elsewhere is NOT thereby exempt — its R2 hostile / accept-set row is required, or a justified `n/a` a reviewer can challenge (delegation is NOT an automatic n/a, and 'delegates to the shared parser' is not itself an acceptable n/a reason). This is the same per-boundary failure-class axis Gate 9 already owns, reached across ticket boundaries — Gate 10 (cross-unit output uniformity) remains disjoint and untouched. It does **not** fire on internal helpers, private functions, or single-call-site refactors (routine-ticket flood prevention) — a ticket that touches no external boundary is graded `n/a` for Gate 9 in full. **Definition**: for EACH Scope-touched external boundary, the ticket MUST carry the **failure-class coverage matrix** — four rows, each yielding >=1 AC OR a one-line `n/a` justification: -- **R1 FULL-DOMAIN INVARIANT**: a property/invariant AC quantifying over the WHOLE valid input domain, INCLUDING boundaries and extremes — min, max, empty, singleton, max-length, and just-inside-each-boundary. (Not a single happy-path fixture: the invariant must hold across the domain, the extremes included.) **Round-trip losslessness (serialization / persistence / file-format / wire-format boundaries)**: when the boundary serializes, persists, or round-trips a value, the R1 invariant MUST include a `parse(serialize(x)) == x` property (and, where the format IS the persisted state, `load(save(x)) == x`) quantified across the whole value domain, INCLUDING the fidelity-prone extremes: an empty key, an empty value, a value that contains the format's own delimiter / separator / quote / newline, a duplicate or accessor (`__proto__`) key, and non-ASCII / unicode. A format that silently drops, truncates, or mangles a value at one of these extremes (a list rendering that cannot represent an `=`-bearing value, a CSV that loses a `__proto__` header column) FAILS R1 even when it is otherwise spec-faithful — a weaker-but-legal format choice is a coverage gap, not an excuse. Where the boundary genuinely cannot round-trip (a one-way digest / hash), R1 is a justified `n/a` naming why. -- **R2 HOSTILE + BOUNDED TERMINATION + RESOURCE-CAP**: malformed / oversized / empty / out-of-domain input yields a **bounded error in bounded time and space** — no hang, no unbounded allocation, no non-error "success" carrying corrupt output. This reuses the Gate 7 overflow-vector + watchdog requirement (a parse-accepted-then-overflows vector through a time-bounded watchdog), boundary-keyed: every external boundary gets its own hostile-input row, not only the one the motivating feature exercised. Where the boundary builds a structure from untrusted input (an object / map keyed by CSV headers, parsed JSON, or form / query / YAML fields), the hostile-input AC MUST also cover hostile KEYS, not only hostile values — prototype-pollution / accessor keys (`__proto__`, `constructor`, `prototype`), duplicate / colliding keys, and empty / non-string keys — asserting no silent column-drop, no prototype mutation, and no swallowed key (the structurally-correct fix is `Object.create(null)` / `Object.defineProperty` / a `Map`). +- **R1 FULL-DOMAIN INVARIANT**: a property/invariant AC quantifying over the WHOLE valid input domain, INCLUDING boundaries and extremes — min, max, empty, singleton, max-length, and just-inside-each-boundary. (Not a single happy-path fixture: the invariant must hold across the domain, the extremes included.) **Round-trip losslessness (serialization / persistence / file-format / wire-format boundaries)**: when the boundary serializes, persists, or round-trips a value, the R1 invariant MUST include a `parse(serialize(x)) == x` property (and, where the format IS the persisted state, `load(save(x)) == x`) quantified across the whole value domain, INCLUDING the fidelity-prone extremes: an empty key, an empty value, a value that contains the format's own delimiter / separator / quote / newline, a duplicate key and an accessor / reserved key the host structure treats specially, and non-ASCII / unicode. A format that silently drops, truncates, or mangles a value at one of these extremes (a list rendering that cannot represent a value containing the format's own delimiter, a serializer that loses an accessor / reserved key) FAILS R1 even when it is otherwise spec-faithful — a weaker-but-legal format choice is a coverage gap, not an excuse. Where the boundary genuinely cannot round-trip (a one-way digest / hash), R1 is a justified `n/a` naming why. +- **R2 HOSTILE + BOUNDED TERMINATION + RESOURCE-CAP**: malformed / oversized / empty / out-of-domain input yields a **bounded error in bounded time and space** — no hang, no unbounded allocation, no non-error "success" carrying corrupt output. This reuses the Gate 7 overflow-vector + watchdog requirement (a parse-accepted-then-overflows vector through a time-bounded watchdog), boundary-keyed: every external boundary gets its own hostile-input row, not only the one the motivating feature exercised. Where the boundary builds a structure from untrusted input (an object / map keyed by parsed headers, parsed JSON, or form / query / YAML fields), the hostile-input AC MUST also cover hostile KEYS, not only hostile values — prototype-pollution / accessor keys (an accessor / reserved key the host structure treats specially), duplicate / colliding keys, and empty / non-string keys — asserting no silent column-drop, no structure-metadata mutation, and no swallowed key (the structurally-correct fix is a null-prototype / own-properties-only container or a Map-equivalent). Beyond hostile values and keys, the row MUST probe **strictness-leniency** wherever a boundary is advertised **strict / canonical / exact / lowercase-only** OR parses a number through a lenient numeric primitive: feed inputs that satisfy the rules' *letter* yet exceed the *advertised* surface — an out-of-alphabet symbol a permissive matcher accepts (a non-ASCII / Unicode-digit / homoglyph numeral a Unicode-aware matcher or a lenient numeric primitive silently accepts before a unit), a sign / whitespace decoration a lenient primitive strips (a leading sign or surrounding-or-embedded whitespace before the digits), and a structurally-valid-but-non-canonical form the canonical writer would never emit (a leading-zero or an explicit-zero-component form) — and assert the enforced boundary equals the advertised one: each is EITHER rejected with the documented error OR normalized and proven idempotent (`parse(format(parse(x))) == parse(x)` AND `format(parse(x))` is canonical). A parser advertised strict that silently accepts a wider surface than it documents is a **strictness-leniency** gap even when no value is corrupted (dogfood evidence: a real dogfood shipped a 'strict' parser that silently accepted out-of-alphabet numerals and non-canonical forms its own fixtures never exercised, none caught by its valid-only fixtures, its Gate 9 matrix, or the panel). - **R3 DESCRIPTION-MATCHES-BEHAVIOR**: runtime behaviour matches the unit's own description / docstring / declared invariant / type annotation — the code does what its own documentation says it does. Concretely, the row's AC MUST RUN the unit through the real public / protocol boundary and assert the observed behaviour against the unit's OWN declared contract (a docstring that claims "returns a sorted copy" → assert the runtime output is a sorted copy AND a distinct object; a declared range / non-null invariant → assert the runtime value obeys it). This is the **EC-SELFDOC** evidence channel (failure mode A, description-vs-behavior drift — see [`../../impl/references/evidence-channels.md`](../../impl/references/evidence-channels.md)); the verifier-side consumer is the `doc-verifier` agent (and the `ac-evaluator` `## Independent Evidence` duty). A row asserting only that the docstring TEXT exists (a grep) without RUNning the unit against it is EC-STATIC on a behavioral claim and does NOT satisfy R3. - **R4 DOC/INTERFACE TRUTHFULNESS**: each advertised example reproduces on a real build, and each advertised boundary equals the enforced boundary. Concretely: (a) for each doc / README / `--help` / man-page worked-example the ticket adds or relies on, the row's AC MUST RUN that command against the real build and diff stdout / exit code against the documented output — byte-for-byte when the output is deterministic, an explicit tolerance otherwise (and the AC names which); (b) for each advertised constraint / limit / range, the AC MUST feed a FORBIDDEN value (just past the advertised limit — MUST be rejected with the documented error) AND an ALLOWED value (just inside the advertised limit — MUST be accepted) to the real boundary, so the enforced boundary is proven equal to the advertised one. This is the **EC-SELFDOC** channel (failure mode E, advertised-boundary != enforced-boundary); the verifier-side consumer is the `doc-verifier` agent under the `.simple-workflow/scratch/` exec carve-out. **Fail-open**: where the boundary advertises no example and no numeric / range limit, R4 is a justified `n/a` (not a FAIL); where the build genuinely cannot be exercised, the AC / verifier records a one-line Caveat rather than force-FAILing. diff --git a/skills/create-ticket/references/agent-spawn-prompts.md b/skills/create-ticket/references/agent-spawn-prompts.md index 80089c7..b343d90 100644 --- a/skills/create-ticket/references/agent-spawn-prompts.md +++ b/skills/create-ticket/references/agent-spawn-prompts.md @@ -98,7 +98,7 @@ Additional context for the planner: - If brief was provided: full brief content (replaces Phase 2 answers). - If findings mode: decomposer-returned skeleton (title, scope_summary, size, depends_on) plus the findings file content, so the planner can lift affected files and observable outcomes verbatim. - **Available capabilities** (verbatim copy of the orchestrator's Pre-computed Context probes): the `Available user skills:` line AND the `Available MCP servers:` line MUST be inlined into the spawn prompt as-is. **MCP inheritance under v8.0.0.** The productive subagents (`implementer`, `planner`, `researcher`, `test-writer`) omit their `tools:` field and therefore inherit the parent session's full tool inventory — including every MCP server configured in `.mcp.json` (project scope) and `~/.claude.json` (user scope), regardless of server name. The orchestrator MUST still serialise the `Available user skills:` and `Available MCP servers:` probes verbatim into the spawn prompt and bind each runtime / visual AC to a concrete capability in the planner's `### Capabilities` table — relevance remains upstream-authoritative. The subagent treats unbound `mcp__*` tools as off-limits even though they appear in its tool inventory (per the agent body's `## Bound Capabilities (Handoff from Orchestrator)` section). The verdict / read-only agents (`ac-evaluator`, `code-reviewer`, `decomposer`, `security-scanner`, `ticket-evaluator`, `tune-analyzer`) retain explicit `tools:` allowlists and do NOT inherit MCP. Empty probe outputs (`(none)`) are passed through unchanged — the planner needs to know whether the gap is "no capability found" vs "not yet probed". -- The literal instruction: "Each AC will be evaluated by an independent evaluator against the canonical AC Quality Criteria at `skills/create-ticket/references/ac-quality-criteria.md`. The planner MUST follow that file as the sole source of truth for Gates 1-10, including the Gate 4 observation-point carve-out, the Gate 5 size-mismatch rationale rule, the Gate 7 oracle-independence requirement for computational ACs, the Gate 8 independent-evidence requirement for behavioral ACs, the Gate 9 failure-class-coverage requirement for Scope-touched external boundaries, and the Gate 10 peer-set-uniformity requirement (assert >=1 unified-convention AC over any >=2-peer sibling set the Scope creates, or a justified n/a). The decomposer's `peer_set:` / `shared_conventions:` hint for this ticket is forwarded below as the upstream peer-set signal. ACs that fail any gate will be rejected." +- The literal instruction: "Each AC will be evaluated by an independent evaluator against the canonical AC Quality Criteria at `skills/create-ticket/references/ac-quality-criteria.md`. The planner MUST follow that file as the sole source of truth for Gates 1-10, including the Gate 4 observation-point carve-out, the Gate 5 size-mismatch rationale rule, the Gate 7 oracle-independence requirement for computational ACs, the Gate 8 independent-evidence requirement for behavioral ACs, the Gate 9 failure-class-coverage requirement for Scope-touched external boundaries, and the Gate 10 peer-set-uniformity requirement (assert >=1 unified-convention AC over any >=2-peer sibling set the Scope creates, or a justified n/a). The decomposer's `peer_set:` / `shared_conventions:` hint for this ticket is forwarded below as the upstream peer-set signal, AND — when this ticket is a member of a cross-ticket `shared_input_boundary` family (the same external input class is consumed by sibling tickets too) — that membership (the shared input class + the sibling ticket ids) is forwarded as the upstream Gate 7 sibling-guard / Gate 9 input-axis signal: when present, each named sibling (this ticket included) MUST carry its OWN hostile / accept-set failure-class row for that input class even if it delegates the input handling to a shared parser created in another ticket. Delegation is NOT an automatic n/a: the row is required, OR a one-line justified n/a a reviewer can challenge — and 'delegates to the shared parser' is NOT an acceptable n/a rationale (delegation is exactly why the row is owed), so an acceptable n/a must name a DIFFERENT reason (e.g. this sibling never re-serializes or round-trips the parsed value, only forwarding an opaque handle). ACs that fail any gate will be rejected." - The literal instruction: "Emit a `### Capabilities` section in the ticket between `### Implementation Notes` and `### Claude Code Workflow`. Each row carries `Name | Type | Purpose | Used by | Bound AC(s)`." - The literal instruction: "Apply Gate 6 of the canonical AC Quality Criteria — every runtime/visual AC (live rendering, console-error count, keyboard focus/hover, WCAG contrast, network I/O, FS-state-dependent) MUST appear in at least one row's `Bound AC(s)` column OR be rewritten as a static AC. If the available capabilities probe returned `(none)` and an AC is runtime/visual, list the gap under `#### Capability Gaps` and rewrite the AC to be static-verifiable." @@ -152,7 +152,7 @@ d. Max 2 rounds (initial + 1 revision). If still FAIL, run the autopilot-policy ### Autopilot-policy escalation (gates.ticket_quality_fail) -After 2 rounds of FAIL, check `{ticket-dir}/autopilot-policy.yaml` at `.simple-workflow/backlog/product_backlog/{parent-slug}/{ticket-dir}/`. If missing **and** `brief=` was given **AND** `brief_mode == auto` (resolved per the canonical precedence rule **chain: precedes mode:** introduced in vX.Y.0: `chain: on` → `brief_mode == auto`, `chain: off` → `brief_mode == manual`; when `chain:` is absent the legacy `mode:` field is read for backward compatibility; super-legacy briefs lacking both keys are treated as `auto` ≡ `chain=on`. The deprecated `mode=` argument supplied to `/brief` is mirrored into the brief frontmatter's `mode:` field, so this fallback path continues to honour pre-vX.Y.0 briefs unchanged. The `mode:` field will be removed in vX.(Y+1).0; the precedence rule converges on the `chain:` reader at that point.), also check `{brief-parent-dir}/autopilot-policy.yaml` (e.g. `.simple-workflow/backlog/briefs/active/{slug}/`). When `brief_mode == manual` (≡ `chain: off`), the brief-parent `autopilot-policy.yaml` fallback is **skipped** — manual-mode runs do not pull retry-strategy from autopilot policy and proceed directly to the interactive flow below. +After 2 rounds of FAIL, check `{ticket-dir}/autopilot-policy.yaml` at `.simple-workflow/backlog/product_backlog/{parent-slug}/{ticket-dir}/`. If missing **and** `brief=` was given **AND** `brief_mode == auto` (resolved per the canonical precedence rule **chain: precedes mode:** introduced in vX.Y.0: `chain: on` → `brief_mode == auto`, `chain: off` → `brief_mode == manual`; when `chain:` is absent the legacy `mode:` field is read for backward compatibility; super-legacy briefs lacking both keys are treated as `auto` ≡ `chain=on`. The deprecated `mode=` argument supplied to `/brief` is mirrored into the brief frontmatter's `mode:` field, so this fallback path continues to honour pre-vX.Y.0 briefs unchanged. The `mode:` field will be removed in v9.0.0; the precedence rule converges on the `chain:` reader at that point.), also check `{brief-parent-dir}/autopilot-policy.yaml` (e.g. `.simple-workflow/backlog/briefs/active/{slug}/`). When `brief_mode == manual` (≡ `chain: off`), the brief-parent `autopilot-policy.yaml` fallback is **skipped** — manual-mode runs do not pull retry-strategy from autopilot policy and proceed directly to the interactive flow below. - If a policy file is present, read `gates.ticket_quality_fail`: - `retry_with_feedback` + retry count < `max_retries` → continue retrying. Print `[AUTOPILOT-POLICY] gate=ticket_quality_fail action=retry_with_feedback round={n}`. diff --git a/skills/create-ticket/references/autopilot-policy-reference.md b/skills/create-ticket/references/autopilot-policy-reference.md index 361fa4f..4b1c2bc 100644 --- a/skills/create-ticket/references/autopilot-policy-reference.md +++ b/skills/create-ticket/references/autopilot-policy-reference.md @@ -5,33 +5,35 @@ This document catalogs the per-ticket knobs that individual skills read from limits, etc.) is documented in `.simple-workflow/docs/plans/` — this page lists only the fields that orchestrator skills consult directly outside the gate machinery. -## `constraints.sonnet_size_threshold` +## Generator model policy -**Consumed by**: `/impl` (Generator model selection, step 13). +simple-workflow does not size-route the generation-side model. The model each +generation-side agent runs on is fixed in its agent-file frontmatter: -**Accepted values**: `S`, `M`, `L`, `off`. - -**Default** (field absent OR policy file absent): `M`. - -**Effect**: - -| Value | Sizes that use sonnet | Sizes that use opus | +| Agent | `model:` | Rationale | |---|---|---| -| `S` | S | M, L, XL, unknown | -| `M` (default) | S, M | L, XL, unknown | -| `L` | S, M, L | XL, unknown | -| `off` | — (none) | all sizes | - -The default `M` preserves the behavior shipped before the knob was -introduced: Size S and M tickets run the Generator on sonnet; L/XL/unknown -escalate to opus. - -Set `constraints.sonnet_size_threshold: off` in a high-risk brief's policy -file to force every ticket under that brief to use opus, regardless of -assigned Size. Set `constraints.sonnet_size_threshold: L` to experiment -with a cheaper Generator on larger tickets — not recommended for -production pipelines; use `M` unless you have measured the failure-rate -trade-off. +| `agents/implementer.md` | `opus` (always) | High-volume work where the retry economy of the stronger model beats size-routing: at the current price ratio (Opus ≈ 1.67× Sonnet per 1M tokens) a single extra evaluation round on a downgraded generator erases the routing saving, so opus is the cost-rational default. | +| `agents/planner.md` | `inherit` | Small-output, high-leverage step — a planning error loses the whole implement + evaluate round. Inheriting the session model lifts the planning ceiling to whatever model the session runs (raising the bar on newest model families). | +| `agents/decomposer.md` | `inherit` | Same shape — a decomposition error is lost per ticket. Inherits the session model. | + +`inherit` resolves to the session model by default. The resolution order is +`CLAUDE_CODE_SUBAGENT_MODEL` env var > a per-invocation override passed by the +caller > the agent frontmatter > the session model. To force the whole +generation + verification fleet onto one model regardless of these per-agent +pins, set `CLAUDE_CODE_SUBAGENT_MODEL` in the environment before launching. + +The evaluation-side model allocation (sonnet by default, opus for +`criticality == critical` or the `exhaustive` tier via the byte-identical +`agents/ac-evaluator-hi.md` sibling) is described in +[`../../impl/references/verification-depth.md`](../../impl/references/verification-depth.md) +`## Criticality scalar` / `## Effects ladder` and is unaffected by this policy. + +> **Migration note** — the per-ticket `constraints` knob that used to +> size-route the Generator model (documented in this section in earlier +> versions) was **removed**; the Generator now always runs on opus per the +> policy above. If an older `autopilot-policy.yaml` still carries that routing +> field, it is now an unknown key with no consumer and is silently ignored — a +> harmless no-op, so no migration action is required. ## `constraints.verification_depth` @@ -270,6 +272,49 @@ switch for the refute-then-synthesize merge; it is independent of `constraints.eval_panel`, `constraints.independent_evidence`, `constraints.oracle_verification`, and `constraints.verification_depth`. +## `constraints.accept_set_conformance` + +**Consumed by**: `/impl` (Step 15 `accept_set_conformance` resolution — the per-AC +deterministic trigger + the EXECUTED accept-set sweep handoff to `ac-evaluator` / +`ac-evaluator-hi`), the `ac-evaluator` `## Failure-class panel` L-ROBUSTNESS lens +(which EXECUTES the sweep for the `triggered-on=` ACs against an independent +hand-coded spec oracle), and the `hooks/accept-set-verify.sh` PostToolUse gate +(which verifies the persisted `## Accept-set sweep` line post-hoc). + +**Accepted values**: `auto`, `off`. + +**Default** (field absent OR policy file absent OR unknown value): `auto` +(fail-safe to active). + +**Effect**: when `auto`, **Advertised-Accept-Set Conformance** (v8.5.0+) is +active. For any AC whose boundary advertises **strict / canonical / lossless / +limit** (word-existence in the AC, its Implementation Notes, the docstring, or +`--help`) OR shares an input class with a `shared_input_boundary` sibling (a +keyed structure built from untrusted input triggers the K axis even with no +lexical word), the orchestrator marks the AC `triggered-on=` and the evaluator +MUST NOT merely reason about the accept set — it EXECUTES a generative +grammar-complement sweep in `.simple-workflow/scratch/` and diffs the unit's +accept-set against an independent hand-coded spec oracle (the four metamorphic +relations MR-FINITE / MR-ALPHABET / MR-CANONICAL / MR-KEYFAITH; see +[`../../impl/references/accept-set-conformance-harness.md`](../../impl/references/accept-set-conformance-harness.md)). +A divergence is FAILed only under the two-tier oracle-authoritative gate (the +oracle is authoritatively narrower than the unit's accept-set), else recorded as +ADVISORY PASS-WITH-CAVEATS (the EC-SELFDOC fail-open posture). The evaluator +persists one `## Accept-set sweep` line per inspected boundary, and +`hooks/accept-set-verify.sh` deterministically verifies that line — a triggered +boundary not run, an alphabet/unicode (A/U-axis) sweep that skipped the astral +complement, or an authoritative divergence not driven to FAIL is BLOCKED, and a +thin A/U corpus is NOTED (advisory only, never blocked — corpus-size is a weak +depth proxy; see CLAUDE.md `SW_ACCEPT_SET_CONFORMANCE_MODE` / `SW_AASC_CORPUS_FLOOR`), +enforced (`on`) by default from v8.5.0 — set `SW_ACCEPT_SET_CONFORMANCE_MODE=metric-only` +for observe-only or `off` to disable the hook. When the policy field `off`, the EXECUTED sweep and the hook +gate stand down: the evaluator verifies the boundary by the prior read-only +strictness reasoning and records a one-line Caveat — the byte-for-byte revert. +This is the per-brief kill switch for the Advertised-Accept-Set Conformance +feature line; it is independent of `constraints.verification_depth`, +`constraints.oracle_verification`, `constraints.eval_panel`, and +`constraints.independent_evidence`. + ## `constraints.selfdoc_verification` **Consumed by**: the EC-SELFDOC evidence channel in diff --git a/skills/create-ticket/references/spec-decomposer-input.md b/skills/create-ticket/references/spec-decomposer-input.md index a693596..336d8d7 100644 --- a/skills/create-ticket/references/spec-decomposer-input.md +++ b/skills/create-ticket/references/spec-decomposer-input.md @@ -66,6 +66,8 @@ The caller (`/create-ticket` orchestrator) is responsible for: Both forms produce the same `## Result` block (status / parent slug / tickets / topological order / rationale) bounded by the agent's Context Conservation Protocol (< 500 tokens). Cycle detection, the 8-ticket cap, and the `Status: partial` overflow path apply identically across forms. +The `## Result` block additionally carries two upstream signals for the planner: the per-ticket `peer_set:` / `shared_conventions:` output-uniformity hint (Gate 10), and ONE mandatory-present cross-ticket `shared_input_boundary:` hint (Gate 7 sibling-guard / Gate 9) naming each external input class consumed by `>=2` DIFFERENT tickets together with the sibling ticket ids — emitted even when each consuming ticket creates only a single unit (per-ticket `peer_set: false`) and even when a consuming ticket merely delegates the input handling to a shared parser created in another ticket. Both signals are advisory and emitted unconditionally; see `agents/decomposer.md` for the exact shape. + ## Failure modes - Missing `Input form:` header → return `Status: failed` with rationale `missing input form header`. diff --git a/skills/create-ticket/references/ticket-template.md b/skills/create-ticket/references/ticket-template.md index 0b4f487..5d8bffb 100644 --- a/skills/create-ticket/references/ticket-template.md +++ b/skills/create-ticket/references/ticket-template.md @@ -36,7 +36,7 @@ Generate the ticket in the following format: | Failure class | AC(s) or n/a justification | |------|------| | R1 FULL-DOMAIN INVARIANT (whole valid domain incl. min/max/empty/singleton/max-length/just-inside-each-boundary) | AC-... \| n/a: ... | -| R2 HOSTILE + BOUNDED TERMINATION + RESOURCE-CAP (malformed/oversized/empty/out-of-domain -> bounded error in bounded time/space; no hang, no non-error success) | AC-... \| n/a: ... | +| R2 HOSTILE + BOUNDED TERMINATION + RESOURCE-CAP (malformed/oversized/empty/out-of-domain -> bounded error in bounded time/space; no hang, no non-error success; when this unit is a member of a cross-ticket `shared_input_boundary` family it owes this row even if it DELEGATES to a shared parser — 'delegates to the shared parser' is not an acceptable n/a) | AC-... \| n/a: ... | | R3 DESCRIPTION-MATCHES-BEHAVIOR (runtime matches the unit's own docstring/declared invariant) | AC-... \| n/a: ... | | R4 DOC/INTERFACE TRUTHFULNESS (each advertised example reproduces on a real build; advertised boundary == enforced boundary) | AC-... \| n/a: ... | diff --git a/skills/impl/SKILL.md b/skills/impl/SKILL.md index a22b4e3..ddb3ab5 100644 --- a/skills/impl/SKILL.md +++ b/skills/impl/SKILL.md @@ -110,7 +110,7 @@ Intra-impl state lives under `phases.impl.*` in `{ticket-dir}/phase-state.yaml`. - `Type = MCP` → `` `ToolSearch query="select:" max_results=1`, then invoke `` directly `` (the `` is the full `mcp____` slug, substituted verbatim from the row's `Name` column). The column exists because plugin subagents see `mcp__*` (and some Skills) as **deferred tools** — names visible, schemas unloaded — so direct invocation raises `InputValidationError` until `ToolSearch` fetches the schema. By generating `How to load` from the `Type` column at orchestrator time, user-added MCP servers (mounted via the user's `.mcp.json` or `~/.claude.json`) and user-installed Skills (under `~/.claude/skills/` or `.claude/skills/`) are handled identically to anything shipped by the plugin without any per-skill code change. The implementer's `## Side-effect ban` carries an explicit advisory-invocation exception for entries on this list. When the ticket has no `### Advisory Capabilities` section (older ticket or empty by Gate 6.5 design), emit `## Advisory capabilities (per ticket): (none)` and proceed. The Advisory table is for productive subagents only — do NOT inline it into `ac-evaluator` spawn prompts at Step 15 (Advisory ≠ verification). - - `subagent_type: simple-workflow:implementer`; description "Implement plan for ". Model per `constraints.sonnet_size_threshold` in `{ticket-dir}/autopilot-policy.yaml` (`S`/`M`/`L`/`off`; default `M`; `off` → opus); see `skills/create-ticket/references/autopilot-policy-reference.md`. + - `subagent_type: simple-workflow:implementer`; description "Implement plan for ". The Generator always runs on opus (`agents/implementer.md` frontmatter `model: opus`); see the **Generator model policy** in `skills/create-ticket/references/autopilot-policy-reference.md`. - Prompt fields a-g: plan path (read full), AC list ("You will be evaluated by an independent evaluator"), investigation, user instructions, round 2+ feedback (`eval-round-{n-1}.md` / `quality-round-{n-1}.md`), CLAUDE.md lint/test ref, round-1 Dry-Run plan. h. KB injection: Read `.simple-workflow/kb/index.yaml`; filter `role=implementer` and `confidence >= 0.8`; include up to 20 summary lines under "## Known Project Patterns". If `.simple-workflow/kb/index.yaml` does not exist, skip silently. **AC always wins over KB patterns on conflict.** i. Autopilot: if `autopilot-policy.yaml` has `constraints.allow_breaking_changes: false`, include "CONSTRAINT: Do not introduce breaking changes to existing public APIs, interfaces, or exported functions." @@ -128,9 +128,9 @@ Intra-impl state lives under `phases.impl.*` in `{ticket-dir}/phase-state.yaml`. > **CHECKPOINT**: Read `phase-state.yaml`, confirm `next_action: start-evaluator`, proceed to Step 15. Do NOT end your turn. -15. **MUST invoke the AC Evaluator agent via the Agent tool** — spawn `simple-workflow:ac-evaluator` (sonnet) by default, OR `simple-workflow:ac-evaluator-hi` (opus) when the Step 3a resolver set `EVALUATOR_MODEL == opus` (M5, v8.3.0+; the per-spawn `model:` override is rejected by the Agent JSONSchema, so the model is selected by which agent file is spawned — `agents/ac-evaluator-hi.md` is byte-identical to `ac-evaluator` except its `name:` and `model:` frontmatter lines). Both agent files honour the identical spawn-prompt contract (fields a-l, `## Bound capabilities (per AC)`, `Oracle verification:`). **NEVER self-assess AC compliance** — Evaluator reads code via `git diff` and renders PASS/FAIL (Ticket 002 failure mode L554-L559). Fail immediately if not invocable. +15. **MUST invoke the AC Evaluator agent via the Agent tool** — spawn `simple-workflow:ac-evaluator` (sonnet) by default, OR `simple-workflow:ac-evaluator-hi` (opus) when the Step 3a resolver set `EVALUATOR_MODEL == opus` (M5, v8.3.0+; the per-spawn `model:` override is rejected by the Agent JSONSchema, so the model is selected by which agent file is spawned — `agents/ac-evaluator-hi.md` is byte-identical to `ac-evaluator` except its `name:` and `model:` frontmatter lines). Both agent files honour the identical spawn-prompt contract (fields a-m, `## Bound capabilities (per AC)`, `Oracle verification:`, `Accept-set conformance:`). **NEVER self-assess AC compliance** — Evaluator reads code via `git diff` and renders PASS/FAIL (Ticket 002 failure mode L554-L559). Fail immediately if not invocable. - **Deterministic capability handoff (per-AC)**: Before constructing the prompt, `Read` `{ticket-dir}/ticket.md` (or the resolved ticket path) and reuse the per-AC bound-capability list extracted from the `### Capabilities` section in Step 13. Inline that mapping verbatim into the spawn prompt under `## Bound capabilities (per AC)` so the Evaluator picks its evidence-gathering capability from the recorded binding instead of re-deriving relevance from the AC text. When the ticket lacks `### Capabilities` (older ticket), fall back to the present ad-hoc path: emit `## Bound capabilities (per AC): (none recorded — ticket pre-dates Gate 6)` and let `ac-evaluator` proceed on its in-house verification methods. The Evaluator MUST treat the recorded binding as authoritative — when an AC carries a bound capability, code inspection alone is not sufficient evidence to PASS that AC; the Evaluator MUST gather live evidence via the bound Skill (or rewrite the AC as static via the planner's `#### Capability Gaps` rule, which is upstream). **Oracle independence (computational ACs, v8.2.0+)**: for any AC whose PASS/FAIL hinges on a computed numeric/algorithmic value, the Evaluator MUST (per `agents/ac-evaluator.md` `## Oracle Independence (computational ACs)`) derive ≥1 expected value from an oracle independent of the code and compare the implementation's RAW (pre-rounding) output with an explicit tolerance — a green project suite is necessary but not sufficient. This applies in single, partition, and multi-verifier modes alike. Inline the `oracle_verification` value resolved at Step 3a into the Evaluator spawn prompt as the field `Oracle verification: {auto|off}` (alongside the `## Bound capabilities (per AC)` handoff) so the Evaluator honours the kill switch deterministically from the prompt rather than reading the policy from disk; when `off`, the Evaluator skips the oracle-independence requirement (pre-v8.2.0 path). **Independent evidence (Gate 8 behavioral ACs, M1, v8.3.0+)**: for every behavioral AC (PASS/FAIL hinges on observable runtime behaviour), the Evaluator MUST (per `agents/ac-evaluator.md` `## Independent Evidence (behavioral ACs)`) establish the AC via at least one evidence channel independent of the implementation's own internals (EC-ORACLE / EC-DIFFERENTIAL / EC-PROPERTY / EC-RUNTIME per `skills/impl/references/evidence-channels.md`), honouring the resolved `EVIDENCE_FLOOR`. Inline the `EVIDENCE_FLOOR` resolved at Step 3a into the Evaluator spawn prompt as the field `Evidence floor: {EC-STATIC+natural|+1-independent|>=2-independent}` (alongside the `Oracle verification: {auto|off}` and `## Bound capabilities (per AC)` handoffs) so the Evaluator honours the floor and the `constraints.independent_evidence` kill switch deterministically from the prompt rather than from disk; when the floor resolves `off`, the Evaluator skips the evidence-floor requirement (pre-v8.3.0 path) but the always-on Gate 7 oracle check still applies when `oracle_verification` is active. Oracle independence (the EC-ORACLE sub-case for computational ACs) is unchanged. **Per-AC oracle audit (M8, v8.4.0+)**: the Evaluator emits one `[ORACLE-AUDIT] ac={id} oracle-kind={first-principles|sibling|hand|none} channels={N} boundary-quantified={y|n}` line per computational / behavioral AC to stderr (per `agents/ac-evaluator.md` `## Oracle Independence (computational ACs)`), surfacing which oracle-kind backed each verdict and how many independent channels it exercised; this observability is UNCONDITIONAL — it emits even under `independent_evidence: off` / `oracle_verification: off` (recording `oracle-kind=none` on the degraded path). **Failure-class panel (v8.4.0+)**: the orchestrator resolves `constraints.eval_panel` (auto|on|off) at Step 3a — `auto` activates the default failure-class eval panel for a ticket touching `>=2` source units OR carrying `>=1` behavioral AC, `on` forces it, `off` disables it (byte-for-byte revert to the single all-purpose pass) — and inlines the resolution into the spawn prompt as field `m` (`--- panel: standard lenses=L-CORRECTNESS,[,] ---`). At `standard` the SINGLE evaluator runs the named lenses SEQUENTIALLY (no added spawn — size gates depth only); at `exhaustive` the panel emphasis rides the three lens spawns. The evaluator emits one `[EVAL-PANEL] lenses={comma-list} mode={single|exhaustive}` line per invocation (M8 observability, beside `[ORACLE-AUDIT]`) UNLESS the panel resolved `off`, and the orchestrator emits `[EVAL-PANEL-MODE] mode={auto|on|off} active={y|n} lenses={N} spawns={1|3} reason={multi-unit|behavioral-ac|forced|trivial|off}` at Step 3a. See [ac-evaluator-orchestration.md](references/ac-evaluator-orchestration.md) `## Default failure-class panel` and `agents/ac-evaluator.md` `## Failure-class panel (default lenses)`. + **Deterministic capability handoff (per-AC)**: Before constructing the prompt, `Read` `{ticket-dir}/ticket.md` (or the resolved ticket path) and reuse the per-AC bound-capability list extracted from the `### Capabilities` section in Step 13. Inline that mapping verbatim into the spawn prompt under `## Bound capabilities (per AC)` so the Evaluator picks its evidence-gathering capability from the recorded binding instead of re-deriving relevance from the AC text. When the ticket lacks `### Capabilities` (older ticket), fall back to the present ad-hoc path: emit `## Bound capabilities (per AC): (none recorded — ticket pre-dates Gate 6)` and let `ac-evaluator` proceed on its in-house verification methods. The Evaluator MUST treat the recorded binding as authoritative — when an AC carries a bound capability, code inspection alone is not sufficient evidence to PASS that AC; the Evaluator MUST gather live evidence via the bound Skill (or rewrite the AC as static via the planner's `#### Capability Gaps` rule, which is upstream). **Oracle independence (computational ACs, v8.2.0+)**: for any AC whose PASS/FAIL hinges on a computed numeric/algorithmic value, the Evaluator MUST (per `agents/ac-evaluator.md` `## Oracle Independence (computational ACs)`) derive ≥1 expected value from an oracle independent of the code and compare the implementation's RAW (pre-rounding) output with an explicit tolerance — a green project suite is necessary but not sufficient. This applies in single, partition, and multi-verifier modes alike. Inline the `oracle_verification` value resolved at Step 3a into the Evaluator spawn prompt as the field `Oracle verification: {auto|off}` (alongside the `## Bound capabilities (per AC)` handoff) so the Evaluator honours the kill switch deterministically from the prompt rather than reading the policy from disk; when `off`, the Evaluator skips the oracle-independence requirement (pre-v8.2.0 path). **Independent evidence (Gate 8 behavioral ACs, M1, v8.3.0+)**: for every behavioral AC (PASS/FAIL hinges on observable runtime behaviour), the Evaluator MUST (per `agents/ac-evaluator.md` `## Independent Evidence (behavioral ACs)`) establish the AC via at least one evidence channel independent of the implementation's own internals (EC-ORACLE / EC-DIFFERENTIAL / EC-PROPERTY / EC-RUNTIME per `skills/impl/references/evidence-channels.md`), honouring the resolved `EVIDENCE_FLOOR`. Inline the `EVIDENCE_FLOOR` resolved at Step 3a into the Evaluator spawn prompt as the field `Evidence floor: {EC-STATIC+natural|+1-independent|>=2-independent}` (alongside the `Oracle verification: {auto|off}` and `## Bound capabilities (per AC)` handoffs) so the Evaluator honours the floor and the `constraints.independent_evidence` kill switch deterministically from the prompt rather than from disk; when the floor resolves `off`, the Evaluator skips the evidence-floor requirement (pre-v8.3.0 path) but the always-on Gate 7 oracle check still applies when `oracle_verification` is active. Oracle independence (the EC-ORACLE sub-case for computational ACs) is unchanged. **Per-AC oracle audit (M8, v8.4.0+)**: the Evaluator emits one `[ORACLE-AUDIT] ac={id} oracle-kind={first-principles|sibling|hand|none} channels={N} boundary-quantified={y|n}` line per computational / behavioral AC to stderr (per `agents/ac-evaluator.md` `## Oracle Independence (computational ACs)`), surfacing which oracle-kind backed each verdict and how many independent channels it exercised; this observability is UNCONDITIONAL — it emits even under `independent_evidence: off` / `oracle_verification: off` (recording `oracle-kind=none` on the degraded path). **Failure-class panel (v8.4.0+)**: the orchestrator resolves `constraints.eval_panel` (auto|on|off) at Step 3a — `auto` activates the default failure-class eval panel for a ticket touching `>=2` source units OR carrying `>=1` behavioral AC, `on` forces it, `off` disables it (byte-for-byte revert to the single all-purpose pass) — and inlines the resolution into the spawn prompt as field `m` (`--- panel: standard lenses=L-CORRECTNESS,[,] ---`). At `standard` the SINGLE evaluator runs the named lenses SEQUENTIALLY (no added spawn — size gates depth only); at `exhaustive` the panel emphasis rides the three lens spawns. The evaluator emits one `[EVAL-PANEL] lenses={comma-list} mode={single|exhaustive}` line per invocation (M8 observability, beside `[ORACLE-AUDIT]`) UNLESS the panel resolved `off`, and the orchestrator emits `[EVAL-PANEL-MODE] mode={auto|on|off} active={y|n} lenses={N} spawns={1|3} reason={multi-unit|behavioral-ac|forced|trivial|off}` at Step 3a. See [ac-evaluator-orchestration.md](references/ac-evaluator-orchestration.md) `## Default failure-class panel` and `agents/ac-evaluator.md` `## Failure-class panel (default lenses)`. **Accept-set conformance handoff (`accept_set_conformance`, v8.5.0+):** read `constraints.accept_set_conformance` from the ticket policy (absent file / field / unknown → `auto`, active). When it is not `off`, compute a per-AC deterministic trigger so the evaluator never re-derives the obligation itself (dogfood46: in-context recognition was non-uniform — a delegating-sibling ticket stood the sweep down): for each AC, mark it triggered when EITHER (1) a case-insensitive lexical scan of the AC text, the ticket's `### Implementation Notes`, and the cited docstring / `--help` / schema matches any of the words `strict`, `canonical`, `lossless`, `limit`; OR (2) the AC's external-input boundary belongs to a `shared_input_boundary` sibling family (the same signal the planner / ticket-evaluator spawns forward — a shared-input sibling triggers even with none of those words present, the "delegation is NOT an automatic n/a" case). Furthermore, when a `shared_input_boundary` sibling builds a KEYED STRUCTURE from untrusted input (a record / map / object / namespace keyed by untrusted keys), that AC MUST be included in `triggered-on=` for the K axis even when no lexical `strict` / `canonical` / `lossless` / `limit` word is present — the keyed-structure boundary is itself the trigger. Inline the result into the Evaluator spawn prompt as the single field `Accept-set conformance: {auto|off} triggered-on={comma-list of triggered AC-ids, or (none)}` (alongside `Oracle verification:` / `Evidence floor:` / `Selfdoc verification:`), and emit `[ACCEPT-SET-TRIGGER] mode={auto|off} triggered={ids|none} basis={lexical|shared-input|both|none}` to stderr. The evaluator MUST run the EXECUTED accept-set sweep for EXACTLY the `triggered-on=` ACs — the orchestrator has already done the recognition, so the evaluator does not re-decide whether to run it. Independent of `verification_depth` / `oracle_verification` / `eval_panel`. See [accept-set-conformance-harness.md](references/accept-set-conformance-harness.md). Count positive ACs (`AC_COUNT`); compute `EVALUATOR_MAX_TURNS = max(60, AC_COUNT * 4)` (field `j`; hard `maxTurns: 200` in frontmatter; e.g. `AC_COUNT = 22` → 88). diff --git a/skills/impl/lib/detect-tautological-assertions.sh b/skills/impl/lib/detect-tautological-assertions.sh index 76c8d90..d1f9bf3 100755 --- a/skills/impl/lib/detect-tautological-assertions.sh +++ b/skills/impl/lib/detect-tautological-assertions.sh @@ -15,10 +15,16 @@ # one line per finding, in the form # R: : — # -# This detector is intentionally grep-based and language/framework-agnostic -# at the regex level. See the rules file's `## Limitations` section for the -# documented out-of-scope cases (variable resolution, type inference, -# cross-file analysis, AST-level algebraic simplification). +# This detector is grep-based and JS/Jest-Vitest-targeted: its regexes match the +# `expect(...).toBe* / toEqual` matcher family and JS numeric literals +# (`Number.MAX_VALUE`, `Infinity`, ...), so it only fires on JavaScript/TypeScript +# tests written for Jest/Vitest. It does NOT detect tautological assertions in +# other languages or test frameworks (pytest, JUnit, Go testing, ...); their +# tautological-assertion / oracle-circularity coverage comes from the +# `ac-evaluator`'s semantic pass (`agents/ac-evaluator.md`), not from this static +# helper. See the rules file's `## Limitations` section for the documented +# out-of-scope cases (variable resolution, type inference, cross-file analysis, +# AST-level algebraic simplification). set -euo pipefail if [ "$#" -lt 1 ]; then diff --git a/skills/impl/references/ac-evaluator-orchestration.md b/skills/impl/references/ac-evaluator-orchestration.md index 135865c..22eb57a 100644 --- a/skills/impl/references/ac-evaluator-orchestration.md +++ b/skills/impl/references/ac-evaluator-orchestration.md @@ -110,7 +110,7 @@ on the raw value + a parse-accepted-overflow vector). Substitute `{i}` ∈ - **V1 — runtime / black-box lens (EC-RUNTIME)**: `--- lens: 1/3 runtime/EC-RUNTIME --- Gather evidence through the REAL public / protocol boundary only: drive the actual CLI, the actual MCP Client over a transport, or the exported public API — never internal handlers reached by reflection or imports the real consumer cannot use. A green project suite is necessary but NOT sufficient: confirm the AC's behaviour is observable at the public boundary, and FAIL any AC whose only evidence is a white-box test that bypasses the schema / serialization / transport layer where real consumers fail.` - **V2 — differential / property lens (EC-DIFFERENTIAL or EC-PROPERTY)**: `--- lens: 2/3 differential-or-property/EC-DIFFERENTIAL,EC-PROPERTY --- Establish evidence INDEPENDENT of the implementation's own output. When a reference implementation of the same contract exists, cross-check the implementation against it (EC-DIFFERENTIAL). Otherwise drive a seeded random sweep over the input space (fixed seed → reproducible) and assert the invariants the output must hold — monotonicity, symmetry, idempotence, round-trip, range/gamut containment (EC-PROPERTY). FAIL an AC whose tests assert only a handful of fixed points the code itself could have produced, with no reference cross-check and no property coverage across the distribution. At thorough / exhaustive, where a SECOND independent ALGORITHM for the same contract exists, compare algorithm-vs-algorithm within tolerance (membership / invariant satisfaction is necessary-not-sufficient — a wrong result can still be in-range); and require a committed, fixed-seed property-fuzz loop across the distribution.` -- **V3 — independent-oracle / targeted-fuzz lens (EC-ORACLE + adversarial)**: `--- lens: 3/3 oracle-or-fuzz/EC-ORACLE --- For any computational AC, independently derive at least one expected value from an oracle that does NOT share the implementation's core (a third-party reference library, a published formula applied from first principles, or a cited hand-computed constant) and compare against the implementation's RAW, pre-rounding output with an explicit tolerance — this is the Gate 7 oracle probe, applied with full force. Additionally fuzz at least one parse-accepted-then-overflows vector (a value the parser ACCEPTS that yields a non-finite / out-of-range intermediate, e.g. `oklch(0.5 1e400 30)`) through the tool under a time-bounded watchdog; FAIL if it hangs or returns a non-error success carrying null / NaN fields. A scratch oracle probe under .simple-workflow/scratch/ is permitted. At thorough / exhaustive for a standard-backed computational AC, derive two mutually-validated oracles (>=1 first-principles) and trust a value only when they agree within tolerance, backed by a committed, fixed-seed seeded fuzz; degrade to one oracle + a Caveat where no second oracle / second algorithm exists.` +- **V3 — independent-oracle / targeted-fuzz lens (EC-ORACLE + adversarial)**: `--- lens: 3/3 oracle-or-fuzz/EC-ORACLE --- For any computational AC, independently derive at least one expected value from an oracle that does NOT share the implementation's core (a third-party reference library, a published formula applied from first principles, or a cited hand-computed constant) and compare against the implementation's RAW, pre-rounding output with an explicit tolerance — this is the Gate 7 oracle probe, applied with full force. Additionally fuzz at least one parse-accepted-then-overflows vector (a value the parser ACCEPTS that yields a non-finite / out-of-range intermediate, e.g. a numeric field whose magnitude overflows to Infinity once arithmetic is applied) through the tool under a time-bounded watchdog; FAIL if it hangs or returns a non-error success carrying null / NaN fields. A scratch oracle probe under .simple-workflow/scratch/ is permitted. At thorough / exhaustive for a standard-backed computational AC, derive two mutually-validated oracles (>=1 first-principles) and trust a value only when they agree within tolerance, backed by a committed, fixed-seed seeded fuzz; degrade to one oracle + a Caveat where no second oracle / second algorithm exists.` The lens header mirrors the `--- partition: /2 ---` convention so the agent recognises its role; the `ac-evaluator` body documents the three @@ -212,10 +212,15 @@ grader systematically under-checks): lens rather than only per computational AC. When the unit builds a structure (object / map / record) from untrusted input (keys from CSV headers, parsed JSON, form / query / YAML fields), this lens also probes hostile KEYS, not only - hostile values: prototype-pollution / accessor keys (`__proto__`, - `constructor`, `prototype`), duplicate / colliding keys, empty / non-string - keys — a silently-dropped column, a mutated prototype, or a swallowed key is a - robustness defect. + hostile values: an accessor / reserved key the host structure treats + specially (the prototype-pollution class), duplicate / colliding keys, empty / + non-string keys — a silently-dropped column, a structure-metadata mutation, or + a swallowed key is a robustness defect. It also probes **strictness-leniency** at any boundary + advertised strict / canonical / exact or parsing a value via a lenient + numeric primitive: an out-of-alphabet symbol a permissive matcher accepts; a + sign / whitespace decoration a lenient primitive strips; and a + structurally-valid-but-non-canonical form the canonical writer would never + emit — an enforced surface wider than the advertised strictness is a defect. - **L-CONTRACT-CONFORMANCE** — does each generated unit's observable behaviour match its OWN stated description / declared schema / documented contract (a function that does not do what its name and doc-comment claim; a tool whose @@ -309,6 +314,32 @@ lenses)`) UNLESS the panel resolved `off`. `eval_panel` is independent of `verification_depth` (which scales tier/depth), `oracle_verification` (the EC-ORACLE sub-case), and `independent_evidence` (the per-AC evidence floor). +**Kill switch `constraints.accept_set_conformance: auto|off`** (absent file / +field / unknown → `auto`, active). Resolved by the orchestrator at `/impl` +Step 15 (the per-AC deterministic trigger) and enforced post-hoc by +`hooks/accept-set-verify.sh`: + +- `auto` — Advertised-Accept-Set Conformance is in force: the orchestrator + computes `triggered-on=` per AC (a boundary lexically advertising **strict / + canonical / lossless / limit**, OR a `shared_input_boundary` sibling — a keyed + structure built from untrusted input triggers the K axis even with no lexical + word), inlines `Accept-set conformance: auto triggered-on={ids}` into the + evaluator spawn, and the evaluator EXECUTES the generative grammar-complement + sweep (black-box, vs an independent hand-coded spec oracle) for exactly those + ACs, persisting one `## Accept-set sweep` line per inspected boundary. +- `off` — DISABLE the EXECUTED sweep: the evaluator verifies the boundary by the + prior read-only strictness reasoning and records a one-line Caveat, and the + `hooks/accept-set-verify.sh` gate has no sweep line to verify — reverting + byte-for-byte to the pre-v8.5.0 read-only path. + +The switch is independent of `verification_depth`, `oracle_verification`, +`eval_panel`, and `independent_evidence`. The post-hoc gate +`hooks/accept-set-verify.sh` ships `on` by default from v8.5.0 (enforce — emit a +PostToolUse `decision:block` on a non-conformant persisted sweep; set +`SW_ACCEPT_SET_CONFORMANCE_MODE=metric-only` to observe-only, `off` to disable — +CLAUDE.md); the EXECUTED sweep itself runs whenever this policy switch is not +`off`. + ## Independent-evidence channels (all evaluator modes) Independent of the verifier count, every `ac-evaluator` invocation — single, diff --git a/skills/impl/references/accept-set-conformance-harness.md b/skills/impl/references/accept-set-conformance-harness.md new file mode 100644 index 0000000..05e7446 --- /dev/null +++ b/skills/impl/references/accept-set-conformance-harness.md @@ -0,0 +1,318 @@ +# Accept-set conformance harness (executed grammar-complement sweep) + +Binding parties: the `ac-evaluator` / `ac-evaluator-hi` (which EXECUTE the sweep +in scratch), and the `implementer` / `test-writer` (which commit the fixed +rejection test to the PRODUCT tests when the sweep finds a leak). This file gives +the **copyable SHAPE** of an executed accept-set conformance sweep so the +`## Failure-class panel` L-ROBUSTNESS accept-set obligation can be satisfied by +transcribing a known-good structure instead of re-deriving it. See +[`evidence-channels.md`](evidence-channels.md) (EC-METAMORPHIC) and +[`test-authoring-guidance.md`](test-authoring-guidance.md) rules 4 / 7 for the +obligations this shape satisfies. + +The load-bearing element is **execution**: a contract that advertises strict / +canonical / lossless / limit is satisfied by READING, and live runs leaked +despite the prose contract existing — the only thing that catches a one-character +validator regression is the evaluator actually RUNNING the complement corpus +black-box against an INDEPENDENT spec oracle. Adding more prose does not raise +the catch-rate; running the sweep does. + +The single worked module below uses **one** neutral, NON-colour domain (a strict +unsigned decimal integer bounded `[0, 255]`) purely to make the shape concrete +and copyable; that boundary is **illustrative only** — illustrative on both the +domain axis and the language axis, carrying no special status. Transcribe the +ROLES (the Grammar Card, the four MRs, the watchdog, the seeded corpus, the +independent oracle, the fail-open degradation) into whatever boundary your AC +actually inhabits. + +## Trigger (when this sweep is mandatory) + +Run the sweep when EITHER holds: +- the boundary's contract lexically advertises **strict / canonical / lossless / + limit** (word-existence in the AC, its Implementation Notes, the docstring, the + declared schema, or a `--help` line); OR +- a sibling unit accepts the **same input class** (the `shared_input_boundary` + peer signal the planner / ticket-evaluator forward). + +It is **always-on** — independent of the verification-depth tier, applied at +`standard` as well — gated only by `constraints.accept_set_conformance` +(`Accept-set conformance: {auto|off}` in the spawn prompt; absent → `auto`). + +## Grammar Card — derive the axes from the AC/spec + +Read the boundary's spec and fill the four axes; each axis drives one MR: + +- **A — Alphabet**: the symbol set the boundary advertises (e.g. ASCII decimal + digits `0-9`). Drives MR-ALPHABET. +- **U — Unicode transform**: any normalization / digit-folding the boundary + applies before validation (e.g. NFC, a digit-folding numeric parse). Drives the + astral span of MR-ALPHABET. +- **W — canonical Writer**: the single canonical output form the boundary + promises (e.g. no leading zero). Drives MR-CANONICAL. +- **K — Keyed structure**: any map / record built from untrusted input. Drives + MR-KEYFAITH. + +## The independent oracle — SPEC-derived, never the unit's own validator + +Hand-code the accept predicate from the SPEC. It MUST NOT call the unit's +validator, and MUST NOT lean on a language primitive that shares the bug class +(`int()` / `Number()` / `str.isdigit()` accept the whole Unicode decimal-digit +property, which is exactly the leak under test) — check character codes by hand. + +```python +# ILLUSTRATIVE ONLY (neutral domain, Python): strict unsigned decimal octet [0,255]. +# Oracle is SPEC-DERIVED and INDEPENDENT of the unit: hand char-code check, +# never int() / str.isdigit(). +def oracle_accepts(s): + if not isinstance(s, str) or len(s) == 0: + return False + for ch in s: + if ord(ch) < 0x30 or ord(ch) > 0x39: # ASCII digits only + return False + if len(s) > 1 and s[0] == '0': # canonical: no leading zero + return False + v = 0 + for ch in s: + v = v * 10 + (ord(ch) - 0x30) + if v > 255: # MR-FINITE: bounded magnitude + return False + return True +``` + +## The four metamorphic relations + the generated corpus + +```python +# ILLUSTRATIVE ONLY: build the complement corpus by PROPERTY, naming no script. +import unicodedata + +def make_corpus(): + corpus = list('0 1 9 10 42 99 100 199 255'.split()) # in-set anchors + corpus += ['256','300','00','01','0255','+1','-1','1.0','1e2','',' ','\t9'] # MR-FINITE / MR-CANONICAL + bmp = astral = 0 + for cp in range(0x110000): # MR-ALPHABET: iterate the codepoint space + if 0x30 <= cp <= 0x39 or 0xD800 <= cp <= 0xDFFF: + continue + ch = chr(cp) + if unicodedata.category(ch) == 'Nd': # select by the decimal-digit PROPERTY + corpus.append(ch) # every script's digits, BMP + astral + astral += (cp > 0xFFFF) + bmp += (cp <= 0xFFFF) + return corpus, bmp, astral +``` + +- **MR-FINITE** — a parse-accepted input whose conversion overflows to a + non-finite / out-of-range intermediate must be rejected (`256`, an + overflowing magnitude). A hang under the watchdog is a FAIL. +- **MR-ALPHABET** — every member of the Unicode decimal-digit property complement + (BMP **and** astral) that the spec's alphabet excludes must be rejected. The + generator names NO script and hard-codes NO codepoint list; a non-zero astral + count is the proof the complement actually crossed the BMP boundary. +- **MR-CANONICAL** — a structurally-valid-but-non-canonical form the canonical + Writer would never emit (`01`, `0255`) must be rejected OR be + normalized-idempotent (`f(f(x)) == f(x)` and `f(x)` is the canonical form). +- **MR-KEYFAITH** — a reserved / accessor / colliding key in a K-axis structure + must not mutate host-structure metadata or be silently dropped. A CONCRETE + round-trip-faithfulness violation (a drop, an overwrite, or a host-metadata + mutation) is subject to the SAME oracle-authoritative two-tier FAIL gating as + the other MRs (see [§ FAIL gating](#fail-gating--oracle-authoritative-two-tier-no-false-positive-storm) + below): a violation on a lossless / strict keyed boundary the oracle + authoritatively narrows is a FAIL; a legitimately-WIDE keyed boundary (arbitrary + keys accepted by contract, last-write-wins, no protected metadata) is advisory + (`authoritative=n`, a Caveat). The copyable shape for this class lives in + [§ The worked MR-KEYFAITH (b) shape](#the-worked-mr-keyfaith-b-shape-reflection-derived-keyed-structure) + below — a reflection-derived key generator paired with an independent + round-trip-faithfulness oracle. + +## The worked MR-KEYFAITH (b) shape reflection-derived keyed structure + +The MR-ALPHABET module above derives its corpus from a **property** (the Unicode +decimal-digit category) and names no script or codepoint. The K-axis class needs +the same move so the self-elicited sweep DERIVES the dangerous keys instead of +naming literals: a hand-picked reserved-key list is the keyed-structure analogue +of hard-coding a per-script digit table — it tests only the keys the author +happened to recall, and the one that ships the leak is the one nobody listed. +This subsection gives MR-KEYFAITH the same copyable structure as MR-ALPHABET, for +a K-axis structure built from untrusted `(key, value)` input pairs. It is +**illustrative only** on both the domain axis and the language axis; transcribe +the ROLES (independent round-trip-faithfulness oracle; reflection-derived +generator; black-box diff), not the surface domain. Gating follows the shared two-tier gate (see the MR-KEYFAITH bullet +above): a concrete drop / overwrite / host-metadata mutation on a lossless / +strict keyed boundary the oracle authoritatively narrows is a FAIL; a +legitimately-wide keyed boundary is advisory. + +The oracle is **round-trip-faithfulness** and is INDEPENDENT of the builder: it +reasons only from the input PAIRS and never calls the builder under test to +decide what is correct. The expectation is +computed from the INPUT PAIRS by last-write-wins; it is NEVER read back out of the +builder (the dogfood-50 trap: +deriving `expected` by re-invoking the builder makes the oracle circular — a +builder that drops a key would "agree with itself"). The round-trip leg below +re-runs the builder only to confirm SERIALIZE then BUILD stability; the truth it is +diffed against stays the pairs-derived mapping, never the builder's own output. + +```python +# ILLUSTRATIVE ONLY (neutral domain, Python): round-trip faithfulness for a +# structure rebuilt from untrusted (key, value) pairs. SPEC-DERIVED and +# INDEPENDENT of the builder: the expectation is computed from the INPUT PAIRS by +# last-write-wins, never by asking the builder what it produced. +# build(pairs) -> the structure under test; get(struct, k) -> value or a +# sentinel if absent; serialize(struct) -> the structure's wire form. +def oracle_faithful(pairs, build, get, serialize, MISSING=object()): + # 1. last-write-wins truth, derived only from the inputs. + expected = {} + for k, v in pairs: + expected[k] = v # later pair for a repeated key wins + built = build(pairs) + # 2. every key the inputs asked for returns ITS value unchanged (no drop / + # overwrite / host-metadata bleed) — read back through the public getter. + for k, want in expected.items(): + if get(built, k) != want: + return False # silent drop or wrong-value overwrite + # 3. round-trip: build(serialize(x)) preserves the same observable mapping. + again = build(serialize(built)) + for k, want in expected.items(): + if get(again, k) != want: + return False + return True +``` + +The generator DERIVES the host structure's own reserved / accessor key names BY +REFLECTION at runtime and names no key literal — exactly as MR-ALPHABET selects +by the decimal-digit PROPERTY and names no script. The reflected set MUST include +the structure's own private / internal slot names (the leading-underscore / +name-mangled / non-public attribute names), not only its public methods and +dunders — a private-slot collision (an input key that shadows the structure's own +internal storage slot) is a real drop / overwrite class, and reflection already +exposes those names, so enumerate the FULL reflected set, never a truncated or +sliced subset. The reflection API calls +(below) are introspection plumbing, not key literals; the dangerous names are +whatever the live host happens to expose, so the source stays decontaminated. + +```python +# ILLUSTRATIVE ONLY: derive the hostile key corpus BY REFLECTION over the live +# host structure — name NO key literal. The accessor / reserved names are read +# off the runtime type at sweep time, so this generator is portable across hosts +# and stays free of any product-specific key denylist. +# +# JS illustration : keys = new Set(Object.getOwnPropertySymbols(target)); +# for (let p = target; p; p = Object.getPrototypeOf(p)) +# Object.getOwnPropertyNames(p).forEach(n => keys.add(n)); +# // climb the ancestor chain via getPrototypeOf to its root, +# // adding each level's own names — names the accessor / +# // reserved set by reflection, no literal. +# Py illustration : as below, via dir(type(target)) over the live type +def reflect_hostile_keys(target): + keys = set() + t = type(target) + keys.update(dir(t)) # the type's own method + dunder names + for base in t.__mro__: # walk the resolution order to the root + keys.update(vars(base).keys()) # each ancestor's own attribute names + return keys + +def make_key_corpus(target, existing_keys): + hostile = reflect_hostile_keys(target) # reflection-derived accessor names + generic = set() + generic.add('') # empty key (generic structural hostile) + if existing_keys: + generic.add(next(iter(existing_keys))) # duplicate of an existing key + # a key that COLLIDES only after the structure's own normalization (the + # structure decides the fold; we derive a colliding partner from an existing + # key rather than naming a literal collision pair). + for k in existing_keys: + if isinstance(k, str) and k != k.casefold(): + generic.add(k.casefold()) # normalized-collision partner + break + return hostile | generic +``` + +Run the REAL builder black-box over `make_key_corpus(...)` as `(key, value)` +pairs (synthetic values, so the only variable is the key), then DIFF against +`oracle_faithful`. A non-faithful build — a reflection-derived accessor name that +overwrites host-structure metadata, an empty / duplicate / normalized-collision +key that silently drops a value, or a round-trip that loses a mapping — is the +catch; name the offending class in Feedback so the producer round can pin it. A +faithful builder returns a clean zero over the whole reflected corpus (no +false-positive storm), because the oracle accepts exactly the last-write-wins +mapping the inputs demanded. + +## Black-box diff under a watchdog (the executed step) + +```python +# ILLUSTRATIVE ONLY: drive the corpus through the REAL boundary, diff vs oracle. +# A SIGKILL watchdog turns a hang into a FAIL (MR-FINITE DoS arm). The seed makes +# any random extension reproducible (mulberry32 in JS; random.Random(seed) here). +def run_sweep(unit_accepts, corpus): + false_accepts, false_rejects, threw = [], [], [] + for s in corpus: + want = oracle_accepts(s) + try: + got = unit_accepts(s) # the REAL public boundary, never an internal handler + except Exception: + threw.append(s); continue + if got and not want: # candidate accept-set DIVERGENCE + false_accepts.append(s) + if (not got) and want: + false_rejects.append(s) + return false_accepts, false_rejects, threw +``` + +## FAIL gating — oracle-authoritative two-tier (NO false-positive storm) + +A non-empty `false_accepts` is only a candidate. Gate the verdict on whether the +hand-coded oracle is AUTHORITATIVE for this boundary: + +- **FAIL** the AC only when the oracle authoritatively reflects a spec that is + STRICTLY NARROWER than the unit's accept-set — the advertised contract is + narrow and explicit (e.g. "ASCII digits only", "no leading zero", + "bounded [0,N]"). A narrow-spec false-accept the oracle rejects is the leak; + name the leaking class in Feedback so the producer round can pin it. Emit + `authoritative=y` on the marker. +- **ADVISORY only (PASS-WITH-CAVEATS, never force-FAIL)** when the advertised + width is NOT unambiguously narrower than what the unit accepts — the boundary + is legitimately or arguably WIDE (Unicode-aware, locale-flexible, + list-format-tolerant). Here the oracle, not the unit, is the suspect: record + the divergence as `[MEDIUM]` Feedback and a Caveat. Emit `authoritative=n`. + +On a CORRECT narrow unit the sweep is a clean zero (the false-positive-storm +absent check the worked example demonstrates by construction). + +## Two-surface retention (who commits what) + +- **Evaluator (read-only)**: the sweep above runs in `.simple-workflow/scratch/` + and is **discarded** after the round; the evaluator cannot write PRODUCT tests. + It records the divergence in Feedback, the persisted `## Accept-set sweep` + report section, and the `[ACCEPT-SET-SWEEP]` marker. +- **Producer (`implementer` / `test-writer`)**: when the evaluator's Feedback + reports an accept-set leak, the next round commits a **fixed rejection + characterization test** to the PRODUCT tests (the leaking input as a RED case; + the validator fix is GREEN) so the regression is locked in the committed suite. + +## No-runnable-artifact caveat (compiled languages) + +When the unit is a COMPILED-language artifact you cannot run black-box (the +evaluator allowlist grants `node` / `python3` but, for Rust/Go, only +`cargo test` / `go test` — no `rustc` / `cargo run` / `go run` / built binary), +you cannot drive the corpus through the real boundary. Run the oracle alone, +record what divergence you CAN derive statically, set +`caveat=no-runnable-artifact` on the marker, and record a one-line Caveat — +never a force-FAIL. + +## Degradation (fail-open) + +Four fail-open arms — the harness never fabricates an oracle and a missing / +non-authoritative oracle is never a FAIL: + +1. **No trigger**: the boundary advertises no strict / canonical / lossless / + limit contract and no same-input-class sibling exists — the sweep is `n/a`. +2. **No runnable artifact**: a compiled unit with no runnable binary — oracle + alone, `caveat=no-runnable-artifact`, Caveat. +3. **No spec**: the domain has no spec to hand-code an oracle from — use whichever + channels DO exist and record a Caveat. +4. **Wide-but-spec-correct**: the spec PERMITS a wider accept-set than the oracle + encodes (the boundary is advertised flexible / Unicode-aware / + locale-tolerant) — do NOT force-FAIL on `false_accepts` the oracle rejects; + the oracle, not the unit, is the suspect. Downgrade to advisory + (`authoritative=n`) and record a Caveat. + +This shape is read at evaluation time to drive an EXECUTED sweep; the producer +half is read at authoring time. diff --git a/skills/impl/references/evidence-channels.md b/skills/impl/references/evidence-channels.md index 8461422..14e57f6 100644 --- a/skills/impl/references/evidence-channels.md +++ b/skills/impl/references/evidence-channels.md @@ -53,9 +53,9 @@ but is NOT independent evidence for a *behavioral* AC on its own. reference implementation of the same contract. *Independent because* a second implementation would have to share the exact same bug to agree. Strongest as **algorithm-vs-algorithm**: when a second, INDEPENDENT ALGORITHM for the same - contract exists (e.g. CSS-MINDE vs chroma-clamping gamut mapping), cross-check + contract exists (two independent algorithms for the same contract), cross-check the two within an explicit tolerance — a membership / invariant test alone - (`inGamut`, "is sorted") is **necessary-not-sufficient** because a wrong + (a containment check, "is sorted") is **necessary-not-sufficient** because a wrong result can still satisfy it. At `thorough` / `exhaustive` use the second-algorithm differential where one is identifiable; degrade to membership / property coverage + a Caveat where no second algorithm exists. @@ -68,6 +68,24 @@ but is NOT independent evidence for a *behavioral* AC on its own. a hand-picked grid — see [`independent-oracle-harness.md`](independent-oracle-harness.md); it degrades to deterministic coverage + a Caveat where no PRNG idiom exists. +- **EC-METAMORPHIC** — a **specialization layered on EC-PROPERTY**: when a boundary + advertises **strict / canonical / lossless / limit** or a same-input-class sibling + exists, an EXECUTED generative grammar-complement sweep whose accept-set is diffed + black-box against an INDEPENDENT hand-coded spec oracle (the four metamorphic + relations MR-FINITE / MR-ALPHABET / MR-CANONICAL / MR-KEYFAITH — see + [`accept-set-conformance-harness.md`](accept-set-conformance-harness.md)). + *Independent because* the oracle derives the spec accept-set without ever calling + the unit's own validator. It is NOT a member of the four-channel naming set + `{EC-ORACLE, EC-DIFFERENTIAL, EC-PROPERTY, EC-RUNTIME}` the Gate 8 binding rule + draws from — a behavioral AC still names one of those four. FAIL gating is + oracle-authoritative two-tier (force-FAIL only on a strictly-narrower advertised + contract; otherwise advisory `[MEDIUM]`), and where no runnable artifact exists it + is **fail-open** with a Caveat. An MR-KEYFAITH divergence (a concrete key drop / + overwrite / host-metadata mutation on a keyed boundary the oracle authoritatively + narrows) is subject to the SAME two-tier oracle-authoritative FAIL gate as the + other relations — force-FAIL when the advertised contract is strictly narrower, + advisory `[MEDIUM]` only where the keyed boundary is legitimately WIDE; it is NOT + a weaker advisory-only floor. - **EC-RUNTIME** — black-box observation through the real public / protocol boundary: the real CLI, the real MCP `Client` over a transport, the exported public API, a rendered DOM — never internal handlers reached by reflection or by imports a real @@ -124,8 +142,8 @@ namespace rule above). - **RT-ABUSE** — misuse of the public contract: out-of-order calls, illegal state transitions, authorization bypass, contract-violating argument combinations. - **RT-MALFORMED** — structurally invalid or adversarial payloads (overlong, mixed - encoding, injection, parse-accepted-then-overflows values such as - `oklch(0.5 1e400 30)`). + encoding, injection, parse-accepted-then-overflows values: an input the parser + ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion). - **RT-EXHAUST** — resource exhaustion: unbounded allocation, pathological recursion, timeouts, rate-limit / quota pressure. - **RT-CONCURRENCY** — races, re-entrancy, shared-state corruption, ordering hazards diff --git a/skills/impl/references/independent-oracle-harness.md b/skills/impl/references/independent-oracle-harness.md index e99920c..5450da7 100644 --- a/skills/impl/references/independent-oracle-harness.md +++ b/skills/impl/references/independent-oracle-harness.md @@ -11,6 +11,14 @@ structure instead of re-deriving it. See [`evidence-channels.md`](evidence-channels.md) (EC-ORACLE / EC-DIFFERENTIAL / EC-PROPERTY) for the obligations this shape satisfies. +The four-part contract below is the canonical, domain-agnostic shape. The single +worked module under each part uses **one** neutral domain (deriving a calendar +date / weekday from a day-number) purely to make the shape concrete and +copyable; that domain is **illustrative only** — it is not a recommended target +and carries no special status. Transcribe the ROLES (a)/(b)/(c)/(d), the +mutual-validation rule, and the fail-open degradation into whatever domain your +AC actually inhabits. + A trustworthy oracle module has FOUR parts. An expected value is trusted only when the first-principles formula and the independent library agree within tolerance — neither alone is enough, because a single library silently shares @@ -18,33 +26,41 @@ its own conventions. ## (a) First-principles block — the spec formula, NO library -Implement the published spec directly, with no dependency on the +Implement the published spec / definition directly, with no dependency on the implementation's core or on any third-party library. This is the channel that is independent of EVERY library's conventions. ```js -// WCAG 2.x relative luminance / contrast, hand-implemented from the spec. -function srgbToLinear(c) { return c <= 0.04045 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4; } -function relativeLuminanceFP(r8, g8, b8) { - const r = srgbToLinear(r8 / 255), g = srgbToLinear(g8 / 255), b = srgbToLinear(b8 / 255); - return 0.2126 * r + 0.7152 * g + 0.0722 * b; -} -function contrastRatioFP(a, b) { - const la = relativeLuminanceFP(...a), lb = relativeLuminanceFP(...b); - const hi = Math.max(la, lb), lo = Math.min(la, lb); - return (hi + 0.05) / (lo + 0.05); +// ILLUSTRATIVE ONLY (neutral domain): proleptic-Gregorian date from a day-number, +// hand-implemented from the published civil-from-days algorithm. No library. +function civilFromDaysFP(z) { // z = days since 1970-01-01 + z += 719468; + const era = Math.floor((z >= 0 ? z : z - 146096) / 146097); + const doe = z - era * 146097; // [0, 146096] + const yoe = Math.floor((doe - Math.floor(doe / 1460) + Math.floor(doe / 36524) - Math.floor(doe / 146096)) / 365); + const y = yoe + era * 400; + const doy = doe - (365 * yoe + Math.floor(yoe / 4) - Math.floor(yoe / 100)); + const mp = Math.floor((5 * doy + 2) / 153); // [0, 11], March-based + const d = doy - Math.floor((153 * mp + 2) / 5) + 1; + const m = mp < 10 ? mp + 3 : mp - 9; + return { year: m <= 2 ? y + 1 : y, month: m, day: d }; } ``` ## (b) Independent-library block — a DIFFERENT library than the SUT -Wrap a reference library that does NOT share the implementation's core (the SUT -uses culori → the oracle uses colorjs.io). This catches spec-implementation -mistakes the first-principles block might also make. +Wrap a reference library that does NOT share the implementation's core (if the +SUT rolls its own date math, the oracle reaches for a maintained datetime +library; if the SUT already depends on that library, pick a different one). This +catches spec-implementation mistakes the first-principles block might also make. ```js -import Color from 'colorjs.io'; -const cjsContrast = (a, b) => new Color(a).contrast(new Color(b), 'WCAG21'); +// ILLUSTRATIVE ONLY: a maintained datetime library as the independent reference. +import { DateTime } from 'luxon'; +const libCivilFromDays = (z) => { + const { year, month, day } = DateTime.fromMillis(z * 86400000, { zone: 'utc' }); + return { year, month, day }; +}; ``` ## (c) Seeded PRNG — reproducible fuzz @@ -62,30 +78,45 @@ function makeRng(seed) { // mulberry32 return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; } -// const rng = makeRng(101); for (let i = 0; i < 1000; i++) { /* assert invariant + oracle agree */ } +// const rng = makeRng(101); +// for (let i = 0; i < 1000; i++) { +// const z = Math.floor((rng() - 0.5) * 2_000_000); // wide, reproducible day-number sweep +// /* assert the invariant holds AND every oracle channel agrees on civilFromDays(z) */ +// } ``` ## (d) Second-algorithm differential helper — algorithm-vs-algorithm Where a SECOND independent ALGORITHM for the same contract exists, expose it so -the test can compare algorithm-vs-algorithm within tolerance — a membership -check (`inGamut`) is necessary-not-sufficient, because a wrong result can still -be in-range. +the test can compare algorithm-vs-algorithm within tolerance (or exactly, for +integer-valued contracts). A bare **membership / range** check (e.g. "the result +is a valid-only value within range" — the analogue of asserting a point is +in-range rather than the right point) is necessary-not-sufficient, because a +wrong result can still land inside the valid set: a containment invariant such as +"the mapped point stays within the allowed range / region" (the kind a gamut or +color-space membership test expresses) catches out-of-range escapes but never a +mere mis-mapping inside the boundary. ```js -// colorjs.io CSS (MINDE) gamut mapping — a DIFFERENT algorithm than culori's clampChroma. -const cjsToGamutCssHex = (input) => - new Color(input).toGamut({ space: 'srgb', method: 'css' }).to('srgb').toString({ format: 'hex' }); -const cjsDeltaEOK = (a, b) => new Color(a).deltaE(new Color(b), { method: 'ok' }); -// expect(cjsDeltaEOK(engineResult, cjsToGamutCssHex(input))).toBeLessThan(0.1); +// ILLUSTRATIVE ONLY: Zeller's congruence — a DIFFERENT algorithm for the weekday +// than counting days-mod-7 off the epoch. Exact agreement is expected (integer contract). +function zellerWeekday({ year, month, day }) { // 0 = Saturday … 6 = Friday + let m = month, y = year; + if (m < 3) { m += 12; y -= 1; } + const K = y % 100, J = Math.floor(y / 100); + return (day + Math.floor(13 * (m + 1) / 5) + K + Math.floor(K / 4) + Math.floor(J / 4) + 5 * J) % 7; +} +const weekdayFromEpoch = (z) => ((((z % 7) + 7) % 7) + 4) % 7; // 1970-01-01 was a Thursday +// const civ = civilFromDaysFP(z); +// expect((zellerWeekday(civ) + 1) % 7).toBe(weekdayFromEpoch(z)); // re-base Zeller's Sat=0 to Sun=0 ``` ## Mutual validation before trust (H1) ```js // The two independent oracles must agree BEFORE either is used as truth: -expect(Math.abs(contrastRatioFP(rgbA, rgbB) - cjsContrast(a, b))).toBeLessThan(2e-3); -// ...only then compare the implementation's RAW (pre-rounding) value against them. +expect(civilFromDaysFP(z)).toEqual(libCivilFromDays(z)); // or toBeCloseTo within tolerance for float contracts +// ...only then compare the implementation's RAW (pre-rounding / pre-canonicalisation) value against them. ``` ## Degradation (fail-open) diff --git a/skills/impl/references/tautological-assertion-rules.md b/skills/impl/references/tautological-assertion-rules.md index 2c70358..a52994d 100644 --- a/skills/impl/references/tautological-assertion-rules.md +++ b/skills/impl/references/tautological-assertion-rules.md @@ -100,10 +100,11 @@ rather than against an INDEPENDENT oracle (a reference library that does not share the implementation's core, a published formula, or a hand-computed truth table). The assertion passes whenever the code is self-consistent, even when the code is wrong, so it cannot observe a correctness regression in the quantity -it claims to check. This is the defect class that let a WCAG contrast solver -accept on a 2-decimal ROUNDED ratio — falsely reporting a target as met — past -a green suite, because every test re-measured with the same rounded value the -code produced. +it claims to check. This is the defect class that lets a test re-derive its +"expected" value from the implementation's own rounded / formatted output and +then assert against it — passing a green suite even when the underlying quantity +is wrong, because every test re-measures with the same rounded value the code +produced. Canonical signature: `assert_close(impl(x), K)` where `K` is derived from `impl` (e.g. `expected = impl.raw(x)`, or `expected = round(impl(x))`), OR the @@ -123,15 +124,15 @@ rule does NOT fire — see Limitations. BAD: - const r = solveForContrast(bg, target); // engine rounds ratio to 2dp - expect(r.ratio).toBeGreaterThanOrEqual(target); // R4: re-thresholds the code's own rounded field - const expected = Math.round(impl.contrast(a, b) * 100) / 100; - expect(impl.contrast(a, b)).toBeCloseTo(expected); // R4: expected derived from the SUT + const r = impl.solve(input, target); // engine rounds its result to 2dp + expect(r.value).toBeGreaterThanOrEqual(target); // R4: re-thresholds the code's own rounded field + const expected = Math.round(impl.compute(x) * 100) / 100; + expect(impl.compute(x)).toBeCloseTo(expected); // R4: expected derived from the SUT GOOD: - import Color from 'colorjs.io'; // independent oracle, no shared core - const oracle = Color.contrast(a, b, 'WCAG21'); - expect(rawContrast(a, b)).toBeCloseTo(oracle, 3); // raw value vs independent oracle, explicit tolerance + import { reference } from 'independent-oracle'; // distinct oracle, no shared core + const oracle = reference(x); + expect(impl.computeRaw(x)).toBeCloseTo(oracle, 3); // raw value vs independent oracle, explicit tolerance ## Hint Expressions diff --git a/skills/impl/references/test-authoring-guidance.md b/skills/impl/references/test-authoring-guidance.md index 22bce75..1583def 100644 --- a/skills/impl/references/test-authoring-guidance.md +++ b/skills/impl/references/test-authoring-guidance.md @@ -5,9 +5,9 @@ file when writing tests). This is the POSITIVE counterpart to [`tautological-assertion-rules.md`](tautological-assertion-rules.md) — that file says what NOT to write; this one says what a STRONG test looks like, scoped by AC type. It exists because a green suite proves the code is self-consistent, not -that it is correct: a dogfood build shipped a WCAG contrast solver that accepted -on a 2-decimal ROUNDED ratio and falsely reported a target as met, past 93 -passing tests, because every test re-measured with the same rounded value the +that it is correct: a dogfood build shipped a computed-value validator that +accepted on a rounded intermediate and falsely reported a target as met, past a +green suite, because every test re-measured with the same rounded value the code itself produced. ## When each technique applies @@ -26,9 +26,9 @@ exit code) needs only a direct assertion. 1. **Independent oracle (computational ACs).** Compare the implementation against an oracle that does NOT share its core: a third-party reference - library (e.g. `colorjs.io` cross-checking a culori-based engine), a published - formula / standard applied from first principles, or a hand-computed truth - table with a cited source. NEVER take the implementation's own output — + library (e.g. a second, independently-authored library cross-checking the + engine under test), a published formula / standard applied from first + principles, or a hand-computed truth table with a cited source. NEVER take the implementation's own output — directly, via an alias, or by re-reading a field the code already rounded — as the expected value. That circularity is rejected by tautological rule R4. At the `thorough` / `exhaustive` depth tier, for a standard-backed @@ -50,12 +50,12 @@ exit code) needs only a direct assertion. 3. **Property / invariant tests.** For math / transform / algorithm code, assert the laws, not just point values: monotonicity, symmetry, idempotence, - round-trip (`decode(encode(x)) == x`), gamut / range containment, and + round-trip (`decode(encode(x)) == x`), range / gamut containment, and conservation. A property holds across a distribution; a point test holds at one point. - When a second INDEPENDENT ALGORITHM for the same contract exists (e.g. - CSS-MINDE vs chroma-clamping gamut mapping, two independent sorts), at the - `thorough` / `exhaustive` depth tier add an **algorithm-vs-algorithm** + When a second INDEPENDENT ALGORITHM for the same contract exists (two + independent algorithms computing the same result, e.g. two independent sorts), + at the `thorough` / `exhaustive` depth tier add an **algorithm-vs-algorithm** differential within an explicit tolerance — a membership / containment check alone is necessary-not-sufficient, because a wrong result can still be in-range. Degrade to property coverage + a note where no second algorithm @@ -63,8 +63,8 @@ exit code) needs only a direct assertion. 4. **Adversarial / non-finite / out-of-range inputs by default.** For any function taking external or untrusted input, include empty, `NaN`, - `Infinity`, negative, zero, overflow, malformed, and out-of-gamut / - out-of-range cases as STANDARD cases, not afterthoughts. These catch DoS + `Infinity`, negative, zero, overflow, malformed, and out-of-range cases as + STANDARD cases, not afterthoughts. These catch DoS hangs (e.g. an unbounded binary search on `Infinity`) and contract-violating outputs (e.g. impossible channel values) that happy-path fixtures miss. @@ -77,8 +77,9 @@ exit code) needs only a direct assertion. tokens (`NaN` / `Infinity` as literal keywords, malformed syntax) that the parser / validator rejects at the door: these exercise the cheap early-return error path, usually already correct. (b) *Parse-accepted-then-overflows* values that pass syntactic parsing but produce a - non-finite / out-of-range INTERMEDIATE deeper in the algorithm — e.g. `oklch(0.5 1e400 30)` - (scientific notation that parses to Infinity chroma), extreme-but-finite magnitudes, denormals, + non-finite / out-of-range INTERMEDIATE deeper in the algorithm — i.e. an input the parser + ACCEPTS that yields a non-finite / out-of-range intermediate after a conversion (e.g. scientific + notation that parses to an Infinity component), extreme-but-finite magnitudes, denormals, or values that overflow only after a multiply. Class (b) is where real DoS hangs and corrupt-success bugs live (an unbounded binary-search / clamp loop on an Infinity intermediate); a suite that tests only class (a) passes green while shipping the class-(b) bug. For every numeric @@ -87,19 +88,23 @@ exit code) needs only a direct assertion. never returns a non-error success carrying null / NaN fields. **Sibling-guard symmetry.** When the function under test shares an input parser / validation - boundary with sibling tools (e.g. several MCP tools that each parse the same color string), the - input-validation guard (finiteness / range / gamut) MUST either live in the SHARED parse / + boundary with sibling tools (e.g. several tools / endpoints parsing the same input class), the + input-validation guard (finiteness / range) MUST either live in the SHARED parse / validation path so every sibling inherits it, OR be replicated AND adversarially tested in EVERY sibling tool that accepts that input class. A guard wired into one tool but absent from its analogous siblings is exactly the `CLAUDE.md ## Modifications` sibling-artifact miss — a dogfood - build added a finite-components guard to the solver but not the analogous `gamut_map` / - `parse_color` tools, shipping a live DoS hang reachable through the unguarded siblings while the - guarded solver passed. Write the class-(b) adversarial test against EACH sibling tool, not only - the one tool the AC names. + build added a finiteness guard to one tool but not the analogous sibling tools parsing the same + input class, shipping a live DoS hang reachable through the unguarded siblings while the + guarded tool passed. Write the class-(b) adversarial test against EACH sibling tool, not only + the one tool the AC names. When the ac-evaluator's EXECUTED accept-set conformance sweep (see + [`accept-set-conformance-harness.md`](accept-set-conformance-harness.md)) reports an accept-set + leak across this shared input class, commit a fixed rejection characterization test for that + leaking input class (leaking input as the RED case, validator fix GREEN) — gated by + `constraints.accept_set_conformance` and always-on, not depth-gated. 5. **Spec-completeness.** Assert every output field and guarantee the spec - promises — a missing field, an absent `inGamut` flag, a dropped `deltaE`, or - a "base color preserved" guarantee that no test checks is an untested + promises — a missing field, an absent status flag, a dropped metric field, or + a "base input preserved" guarantee that no test checks is an untested contract. Enumerate the promised outputs and assert each. 6. **Black-box over white-box.** Exercise the public / protocol boundary (the diff --git a/skills/impl/references/verification-depth.md b/skills/impl/references/verification-depth.md index 5c46e0c..b1da96b 100644 --- a/skills/impl/references/verification-depth.md +++ b/skills/impl/references/verification-depth.md @@ -82,27 +82,36 @@ evaluator model, and leaves the red-team budget at 0. The matrix above scales depth by blast-radius and autonomy. It does NOT account for **correctness criticality** — a tiny `S` ticket can still ship a -wrong-but-self-consistent computed value (the WCAG rounded-meet defect class). +wrong-but-self-consistent computed value (the rounded-meet defect class — a +value that matches the implementation's own rounded output yet is wrong). The criticality floor adds that axis: when a ticket contains at least one **computational AC** (PASS/FAIL hinges on a computed numeric/algorithmic value — see Gate 7 in `skills/create-ticket/references/ac-quality-criteria.md`) in a -**critical domain** — accessibility / security / money / data-integrity / -standard-compliance — the resolved tier is floored at `thorough` regardless of +**critical domain**, the resolved tier is floored at `thorough` regardless of Size × `risk_tolerance`, so the `/audit` skeptical third-pass is forced even on an `S` / `conservative` ticket. The critical-domain determination is a -model-judgment read of the AC text (there is no deterministic ticket field); -keyword cues that should trigger it: WCAG / contrast / a11y / focus-order -/ color-space / gamut / OKLab / luminance / chroma conversion (accessibility & -color-science); auth / crypto / token / signature / input-validation -(security); currency / decimal / rounding / money (money); checksum / hash / -dedup / referential-integrity (data-integrity); RFC / ISO / spec-conformance -(standard-compliance); and any **computational AC over a shared-core input -boundary** — a parser / validation / constant (e.g. an epsilon / range / gamut -guard) shared with sibling tools, where an inconsistency in one sibling is the -exact wrong-but-self-consistent defect class the floor targets. The shared-core -trigger is independent of the named domains above: a computational AC that reads -or must hold an invariant across a shared input boundary floors `critical` even -when its surface domain is otherwise routine. +model-judgment read of the AC text (there is no deterministic ticket field). + +The PRIMARY trigger is a **property**, not membership in a named-domain list: +any **computational AC over a shared-core input boundary** — a parser / +validation / constant (e.g. an epsilon / range / bounds guard) shared with +sibling tools, where an inconsistency in one sibling is the exact +wrong-but-self-consistent defect class the floor targets — floors `critical`. +This shared-boundary property is independent of surface domain: a computational +AC that reads or must hold an invariant across a shared input boundary floors +`critical` even when its domain is otherwise routine. Equally, an AC whose +computed value is **correctness-critical** — a wrong result carries outsized +real-world cost, or the value must conform to a published spec / standard — +floors `critical`. + +The domain families below are **illustrative and non-exhaustive**: they help the +model recognise the property above, they do NOT bound it. A ticket that matches +the property but names none of these still floors `critical`; a ticket that +mentions a cue but carries no computational / correctness-critical AC does not. +Balanced examples — accessibility (e.g. contrast / color-space ratios); +security (e.g. auth / crypto / input-validation); money (e.g. currency / decimal +rounding); data-integrity (e.g. checksum / referential-integrity); and +standard-compliance (e.g. RFC / ISO conformance). The floor only RAISES the tier (`standard` → `thorough`); it never lowers a tier the matrix already resolved higher (`exhaustive` stays `exhaustive`). It is @@ -259,9 +268,9 @@ The resolver also fills two struct fields the floor RAISES: - **evaluator model**: today the evaluator is hardcoded sonnet (`skills/impl/SKILL.md` Step 15 "always sonnet"; `agents/ac-evaluator.md` frontmatter `model: sonnet`). M5 resolves `evaluator_model` to `opus` when `criticality == critical` OR - `depth_tier == exhaustive`, symmetric with the EXISTING size-aware generator-model - selection (`constraints.sonnet_size_threshold` escalates the generator to opus for - L/XL — `skills/create-ticket/references/autopilot-policy-reference.md`). **Platform + `depth_tier == exhaustive` (the Generator already always runs on opus — see the + **Generator model policy** in + `skills/create-ticket/references/autopilot-policy-reference.md`). **Platform caveat**: the Agent tool's JSONSchema does NOT accept a per-invocation `model:` override — the SAME Strategy-B limitation that forces the soft turn budget instead of a per-spawn `maxTurns` (`skills/impl/references/ac-evaluator-orchestration.md` diff --git a/tests/test-ac-evaluator-static-rules.sh b/tests/test-ac-evaluator-static-rules.sh index f741ecb..9428552 100755 --- a/tests/test-ac-evaluator-static-rules.sh +++ b/tests/test-ac-evaluator-static-rules.sh @@ -97,9 +97,11 @@ else TESTS_FAILED=$((TESTS_FAILED + 1)) fi -# NAC #4: language/framework names must not appear outside example blocks. +# NAC #4: language/framework names must not appear in PROSE (outside fenced +# example blocks). Fenced code blocks are exempt — a worked example may legitimately +# carry a runner token — mirroring the fence-aware scan in hooks/pre-write-safety.sh. TESTS_TOTAL=$((TESTS_TOTAL + 1)) -NAC4_HITS=$(grep -ciE 'vitest|jest|pytest|junit' "$RULES_FILE" || true) +NAC4_HITS=$(awk '/^[[:space:]]*```/{f=1-f;next} !f{print}' "$RULES_FILE" | grep -ciE 'vitest|jest|pytest|junit' || true) if [ "$NAC4_HITS" -eq 0 ]; then echo -e " ${GREEN}PASS${NC} NAC #4: rules file is framework-agnostic" TESTS_PASSED=$((TESTS_PASSED + 1)) diff --git a/tests/test-accept-set-verify.sh b/tests/test-accept-set-verify.sh new file mode 100755 index 0000000..219266b --- /dev/null +++ b/tests/test-accept-set-verify.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# Behaviour tests for hooks/accept-set-verify.sh — the deterministic +# Advertised-Accept-Set Conformance (AASC) post-hoc gate over the persisted +# `## Accept-set sweep` section of an eval-round-{n}.md report. +# +# Mirrors the proof-by-construction matrix: every conformance predicate +# (P1 stand-down / P2 shallow-astral / P3 sliced-corpus / P4 gating-consistency) +# is exercised against fixture reports, with the no-runnable-artifact escape, +# the A/U-axis floor scoping (K/W small reflection corpora must NOT false-trip), +# the dogfood50 clean shape (NO false positive), the skeleton / n/a / non-eval +# fail-OPEN gates, the kill-switch tri-state, and the jq-absent fail-OPEN path. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/test-helper.sh" + +HOOK="$HOOK_DIR/accept-set-verify.sh" +FIXDIR=$(mktemp -d) + +echo "=== accept-set-verify.sh Tests ===" +echo "" + +mkfix() { printf '%s\n' "$2" > "$FIXDIR/$1"; } + +run_fix() { # run_fix + local json + json=$(jq -n --arg fp "$FIXDIR/$1" '{"tool_input":{"file_path":$fp}}') + run_hook "$HOOK" "$json" +} + +pass_local() { echo -e " ${GREEN}PASS${NC} $1"; TESTS_PASSED=$((TESTS_PASSED + 1)); TESTS_TOTAL=$((TESTS_TOTAL + 1)); } +fail_local() { + echo -e " ${RED}FAIL${NC} $1" + echo -e " RC=$LAST_EXIT_CODE OUT=[$LAST_STDOUT] ERR=[$LAST_STDERR]" + TESTS_FAILED=$((TESTS_FAILED + 1)); TESTS_TOTAL=$((TESTS_TOTAL + 1)) +} + +expect_block() { if [ "$LAST_EXIT_CODE" -eq 0 ] && echo "$LAST_STDOUT" | grep -q '"decision": "block"'; then pass_local "$1"; else fail_local "$1"; fi; } +expect_noblock() { if [ "$LAST_EXIT_CODE" -eq 0 ] && ! echo "$LAST_STDOUT" | grep -q '"decision"'; then pass_local "$1"; else fail_local "$1"; fi; } +expect_metric() { if [ "$LAST_EXIT_CODE" -eq 0 ] && echo "$LAST_STDERR" | grep -qF '[ACCEPT-SET-VERIFY] metric-only: would block'; then pass_local "$1"; else fail_local "$1"; fi; } +expect_silent() { if [ "$LAST_EXIT_CODE" -eq 0 ] && [ -z "$LAST_STDOUT" ] && [ -z "$LAST_STDERR" ]; then pass_local "$1"; else fail_local "$1"; fi; } +# P3 thin-corpus is ADVISORY (dogfood51): a stderr note, NEVER a block. +expect_advisory() { if [ "$LAST_EXIT_CODE" -eq 0 ] && echo "$LAST_STDERR" | grep -qF '[ACCEPT-SET-VERIFY] advisory:' && ! echo "$LAST_STDOUT" | grep -q '"decision"'; then pass_local "$1"; else fail_local "$1"; fi; } + +# ---- Fixtures ------------------------------------------------------------- +mkfix eval-round-F1.md "## Status: FAIL + +## Accept-set sweep +boundary=K triggered=y ran=n astral=n corpus-size=0 divergences=0 authoritative=y caveat=none" + +mkfix eval-round-F2.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=y astral=n corpus-size=300 divergences=0 authoritative=n caveat=none" + +mkfix eval-round-F3.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=y astral=n corpus-size=0 divergences=0 authoritative=n caveat=no-runnable-artifact" + +mkfix eval-round-F3b.md "## Status: PASS-WITH-CAVEATS + +## Accept-set sweep +boundary=A triggered=y ran=n astral=n corpus-size=0 divergences=0 authoritative=n caveat=no-runnable-artifact" + +mkfix eval-round-F4.md "## Status: PASS + +## Accept-set sweep +boundary=U triggered=y ran=y astral=y corpus-size=12 divergences=0 authoritative=n caveat=none" + +mkfix eval-round-F5.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=y astral=y corpus-size=750 divergences=2 authoritative=y caveat=none" + +mkfix eval-round-F6.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=y astral=y corpus-size=750 divergences=0 authoritative=y caveat=none" + +mkfix eval-round-F7.md "## Status: PASS-WITH-CAVEATS + +## Accept-set sweep +boundary=W triggered=n ran=n astral=n corpus-size=0 divergences=0 authoritative=n caveat=none +boundary=K triggered=y ran=y astral=n corpus-size=37 divergences=0 authoritative=n caveat=none +boundary=A triggered=y ran=y astral=y corpus-size=512 divergences=3 authoritative=n caveat=none" + +mkfix eval-round-F7b.md "## Status: PASS + +## Accept-set sweep +n/a (no external-input boundary in scope)" + +mkfix eval-round-F9.md "## Status: IN_PROGRESS + +- [ ] AC-1: do the thing" + +mkfix eval-round-F10.md "## Status: PASS + +- [x] AC-1: done" + +mkfix some-other-file.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=n astral=n corpus-size=0 divergences=0 authoritative=y caveat=none" + +# dogfood51-derived fixtures. +# Fcap: a mis-cased `## Accept-set Sweep` header (capital S) must still be READ +# (case-insensitive Gate 2b) so a P1 stand-down under it is caught, not skipped. +mkfix eval-round-Fcap.md "## Status: FAIL + +## Accept-set Sweep +boundary=K triggered=y ran=n astral=n corpus-size=0 divergences=0 authoritative=y caveat=none" + +# Fthin: the dogfood51 002-r1 shape — a thin-but-conformant A sweep on a PASS +# report (A=5, astral=y, divergences=0). P3 is ADVISORY now, so this must NOT +# block (the false-trip the confirmation dogfood surfaced). +mkfix eval-round-Fthin.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=y astral=y corpus-size=5 divergences=0 authoritative=y caveat=none" + +# Fsuffix: a descriptive corpus-size (dogfood51 001-r1: `5-canonical-forms`) — the +# leading integer is parsed so the annotation cannot dodge the P3 advisory. +mkfix eval-round-Fsuffix.md "## Status: PASS + +## Accept-set sweep +boundary=A triggered=y ran=y astral=y corpus-size=5-canonical-forms divergences=0 authoritative=n caveat=none" + +# Fhash: a mis-LEVELED header (`### Accept-set sweep`, three hashes) must still be +# READ (Gate 2b + section awk match any hash depth) so a P1 stand-down under it is +# caught, not skipped. Live-observed in the dogfood51 ground-truth (003 eval-round-3). +# Same drift class as Fcap, on the hash-depth axis. +mkfix eval-round-Fhash.md "## Status: FAIL + +### Accept-set sweep +boundary=K triggered=y ran=n astral=n corpus-size=0 divergences=0 authoritative=y caveat=none" + +# Ftab: a tab-separated sweep line must still have its fields extracted (field_of +# value class is whitespace-bounded, not space-only) so a P1 stand-down is caught. +printf '## Status: FAIL\n\n## Accept-set sweep\nboundary=K\ttriggered=y\tran=n\tastral=n\tcorpus-size=0\tdivergences=0\tauthoritative=y\tcaveat=none\n' > "$FIXDIR/eval-round-Ftab.md" + +# Fcaps: an ALL-CAPS header (`## ACCEPT-SET SWEEP`) must still be READ — Gate 2b is +# grep -i and the section awk lowercases via tolower(), so the two header-matchers +# accept the SAME set (any case, any hash depth); a bad sweep under it cannot escape. +mkfix eval-round-Fcaps.md "## Status: FAIL + +## ACCEPT-SET SWEEP +boundary=K triggered=y ran=n astral=n corpus-size=0 divergences=0 authoritative=y caveat=none" + +# ---- MODE=on (enforce) ---------------------------------------------------- +echo "--- MODE=on (enforce: decision:block on violation) ---" +export SW_ACCEPT_SET_CONFORMANCE_MODE=on SW_AASC_CORPUS_FLOOR=256 +run_fix eval-round-F1.md; expect_block "F1 P1 stand-down (triggered=y ran=n) -> block" +run_fix eval-round-F2.md; expect_block "F2 P2 shallow-astral (A axis) -> block" +run_fix eval-round-F3.md; expect_noblock "F3 no-runnable-artifact escape (ran=y) -> no block" +run_fix eval-round-F3b.md; expect_noblock "F3b no-runnable-artifact (ran=n, legit degradation) -> no block" +run_fix eval-round-F4.md; expect_advisory "F4 thin corpus (U axis, 12<256) -> ADVISORY only, NOT block (P3 demoted, dogfood51)" +export SW_AASC_CORPUS_FLOOR=8 +run_fix eval-round-F4.md; expect_noblock "F4 floor=8 -> no advisory, silent (12>=8, SW_AASC_CORPUS_FLOOR tunable)" +export SW_AASC_CORPUS_FLOOR=256 +run_fix eval-round-F5.md; expect_block "F5 P4 gating-inconsistency (auth=y div=2 PASS) -> block" +run_fix eval-round-F6.md; expect_noblock "F6 dogfood50 clean shape -> NO false trip" +run_fix eval-round-F7.md; expect_noblock "F7 K small-corpus+astral=n & auth=n divergences -> no trip" +run_fix eval-round-F7b.md; expect_noblock "F7b n/a fallback -> no block" +run_fix eval-round-F9.md; expect_noblock "F9 skeleton IN_PROGRESS (Gate 2a) -> no block" +run_fix eval-round-F10.md; expect_noblock "F10 terminal, no sweep section (Gate 2b) -> no block" +run_fix some-other-file.md; expect_noblock "F8 non-eval path (Gate 1) -> no block" +run_fix eval-round-Fcap.md; expect_block "Fcap capital-S header + P1 -> READ case-insensitively + block (dogfood51 fix #2)" +run_fix eval-round-Fthin.md; expect_advisory "Fthin thin-but-conformant A=5 astral=y PASS report -> advisory, NO block (dogfood51 002-r1 false-trip fix)" +run_fix eval-round-Fsuffix.md; expect_advisory "Fsuffix descriptive corpus-size (5-canonical-forms) -> leading-int parsed -> advisory (dogfood51 fix #3)" +run_fix eval-round-Fhash.md; expect_block "Fhash mis-leveled header (### Accept-set sweep) + P1 -> READ at any hash depth + block (live 003 eval-round-3 escape)" +run_fix eval-round-Ftab.md; expect_block "Ftab tab-separated sweep line + P1 -> fields extracted (whitespace-bounded class) + block" +run_fix eval-round-Fcaps.md; expect_block "Fcaps all-caps header (## ACCEPT-SET SWEEP) + P1 -> READ (awk tolower == Gate2b -i) + block" + +# ---- MODE=metric-only (explicit downgrade; observe only) ------------------ +echo "--- MODE=metric-only (explicit: observe, never block) ---" +export SW_ACCEPT_SET_CONFORMANCE_MODE=metric-only SW_AASC_CORPUS_FLOOR=256 +run_fix eval-round-F1.md; expect_metric "F1 -> metric-only would-block stderr" +run_fix eval-round-F2.md; expect_metric "F2 -> metric-only would-block stderr" +run_fix eval-round-F5.md; expect_metric "F5 -> metric-only would-block stderr" +run_fix eval-round-F6.md; expect_silent "F6 clean -> silent (no metric line)" +run_fix eval-round-F7.md; expect_silent "F7 -> silent (no false trip)" +run_fix eval-round-Fthin.md; expect_advisory "Fthin metric-only -> advisory note (P3 is advisory in BOTH modes, never would-block)" + +# ---- MODE=off (explicit opt-out) ------------------------------------------ +echo "--- MODE=off (explicit opt-out: silent) ---" +export SW_ACCEPT_SET_CONFORMANCE_MODE=off SW_AASC_CORPUS_FLOOR=256 +run_fix eval-round-F1.md; expect_silent "F1 off -> silent exit 0" +run_fix eval-round-F5.md; expect_silent "F5 off -> silent exit 0" + +# ---- MODE=unknown (collapses to metric-only) ------------------------------ +echo "--- MODE=typo (unknown collapses to metric-only) ---" +export SW_ACCEPT_SET_CONFORMANCE_MODE=enforce SW_AASC_CORPUS_FLOOR=256 +run_fix eval-round-F1.md; expect_metric "F1 unknown-mode -> metric-only (not enforce, not off)" +unset SW_ACCEPT_SET_CONFORMANCE_MODE SW_AASC_CORPUS_FLOOR + +# ---- DEFAULT (no env var) -> on (v8.5.0 promotion) ------------------------ +echo "--- DEFAULT (env unset) -> on: the hook ENFORCES by default (v8.5.0) ---" +run_fix eval-round-F1.md; expect_block "F1 with NO env var -> on by DEFAULT (v8.5.0 promotion) -> block" +run_fix eval-round-Fthin.md; expect_advisory "Fthin with NO env var -> on default, thin corpus -> advisory (not block)" +run_fix eval-round-F6.md; expect_noblock "F6 with NO env var -> on default, clean -> no block" + +# ---- jq-absent (fail-OPEN) ------------------------------------------------ +echo "--- jq-absent (fail-OPEN exit 0) ---" +RBIN=$(mktemp -d) +for c in bash cat grep sed awk head tr basename printf; do + p=$(command -v "$c" 2>/dev/null) && ln -s "$p" "$RBIN/$c" 2>/dev/null +done +jq_json=$(printf '{"tool_input":{"file_path":"%s"}}' "$FIXDIR/eval-round-F1.md") +if printf '%s' "$jq_json" | PATH="$RBIN" SW_ACCEPT_SET_CONFORMANCE_MODE=on bash "$HOOK" >/dev/null 2>&1; then + jqrc=0 +else + jqrc=$? +fi +LAST_EXIT_CODE="$jqrc"; LAST_STDOUT=""; LAST_STDERR="" +if [ "$jqrc" -eq 0 ]; then pass_local "jq-absent -> fail-OPEN exit 0"; else fail_local "jq-absent -> fail-OPEN exit 0"; fi +rm -rf "$RBIN" + +# ---- cleanup + summary ---------------------------------------------------- +rm -rf "$FIXDIR" +print_summary diff --git a/tests/test-autopilot-continue.sh b/tests/test-autopilot-continue.sh index f218d11..be3d1cd 100644 --- a/tests/test-autopilot-continue.sh +++ b/tests/test-autopilot-continue.sh @@ -1322,5 +1322,52 @@ else fi cleanup_test_repo +# ============================================================ +# AC-NESTED (proposal 3 / ST-01): nested-form scout in_progress — block stop. +# WI-3 schema-tolerance regression guard. The prior flat-only grep in the +# continuation driver matched `scout: in_progress` but NOT the nested +# `scout:\n status: in_progress` shape, so a purely nested state yielded +# ACTIVE_STEPS=0 and allowed a premature stop. Routing ACTIVE_STEPS/NEXT_STEP +# through parse_active_steps must now block. Reverting that refactor makes this +# case FAIL (decision != block). +# ============================================================ +echo "--- AC-NESTED: nested-form scout in_progress (WI-3) ---" + +setup_test_repo +create_state_file "test-slug" "version: 1 +slug: test-slug +started: 2026-04-15T00:00:00Z +execution_mode: single +total_tickets: 1 +tickets: + - logical_id: test-slug + ticket_dir: 001-test + status: in_progress + steps: + create-ticket: + status: completed + scout: + status: in_progress + impl: + status: pending + ship: + status: pending" + +run_autopilot_hook '{"session_id":"test-nested"}' "$TEST_REPO" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +DECISION=$(echo "$LAST_STDOUT" | jq -r '.decision // ""' 2>/dev/null || echo "") +REASON_NEXT=$(echo "$LAST_STDOUT" | jq -r '.reason // ""' 2>/dev/null | grep -c 'scout' || true) +if [ "$DECISION" = "block" ] && [ "$REASON_NEXT" -ge 1 ]; then + echo -e " ${GREEN}PASS${NC} nested-form scout in_progress: decision=block, next step 'scout' resolved (WI-3 nested tolerance)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} nested-form scout in_progress: expected decision=block with next step 'scout'" + echo -e " Exit code: $LAST_EXIT_CODE, Decision: '$DECISION', scout-in-reason: '$REASON_NEXT'" + echo -e " Stdout: $LAST_STDOUT" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi +rm -f /tmp/.autopilot-continue-test-nested +cleanup_test_repo + echo "" print_summary diff --git a/tests/test-hooks-lib.sh b/tests/test-hooks-lib.sh index 204b165..ec59a52 100755 --- a/tests/test-hooks-lib.sh +++ b/tests/test-hooks-lib.sh @@ -1049,9 +1049,14 @@ printf 'phases:\n scout: {status: completed}\n impl: {status: completed}\n sh sa_ac5_out="$(bash -c "source '$SA_PATH' && resolve_active_state_file '$SA_T5'")" assert_eq "AC-5: rejects done-incomplete (empty stdout)" "" "$sa_ac5_out" -# AC-6: HOOK_OWNED_FIELDS empty by default +# AC-6 (proposal 4 / Foundation 3): HOOK_OWNED_FIELDS ships with exactly ONE +# entry, .runtime_metrics (the append-only telemetry list). The resulting DENY is +# gated by SW_STATE_FIELD_GUARD_MODE in pre-write/pre-edit (default metric-only), +# so a non-empty registry does not change the shipped allow-by-default behaviour. sa_ac6_count="$(bash -c "source '$SA_PATH' && echo \${#HOOK_OWNED_FIELDS[@]}")" -assert_eq "AC-6: registry empty by default" "0" "$sa_ac6_count" +assert_eq "AC-6: registry ships with exactly 1 entry (.runtime_metrics)" "1" "$sa_ac6_count" +sa_ac6_key="$(bash -c "source '$SA_PATH' && printf '%s\n' \"\${!HOOK_OWNED_FIELDS[@]}\"")" +assert_eq "AC-6: the registered field is .runtime_metrics" ".runtime_metrics" "$sa_ac6_key" # AC-7: is_hook_owned_field returns 1 on unknown set +e @@ -1126,6 +1131,19 @@ sa_ac13_exit=$? set -e assert_exit_nonzero "AC-13: initial set is allowed" "$sa_ac13_exit" +# AC-OWN-RM (proposal 4 / Foundation 3): with the SHIPPED registry (no temp +# injection), state_field_change_blocked BLOCKS a .runtime_metrics change and +# echoes the violated key on stdout; a non-owned field change still allows. +set +e +sa_ownrm_field="$(bash -c "source '$SA_PATH'; state_field_change_blocked /tmp/x 'runtime_metrics: []' 'runtime_metrics: [{boundary: x}]'")" +sa_ownrm_exit=$? +bash -c "source '$SA_PATH'; state_field_change_blocked /tmp/x 'status: pending' 'status: completed'" >/dev/null 2>&1 +sa_ownrm_other_exit=$? +set -e +assert_exit_zero "AC-OWN-RM: shipped registry blocks a .runtime_metrics change" "$sa_ownrm_exit" +assert_eq "AC-OWN-RM: blocked field echoed is .runtime_metrics" ".runtime_metrics" "$sa_ownrm_field" +assert_exit_nonzero "AC-OWN-RM: a non-owned field change still allows" "$sa_ownrm_other_exit" + # Negative AC-1: exactly 3 new public functions + HOOK_OWNED_FIELDS public var sa_neg_ac1_before_funcs="$(declare -F | awk '{print $3}' | sort -u)" # shellcheck disable=SC1090 @@ -1135,11 +1153,13 @@ sa_neg_ac1_new_funcs="$(comm -13 <(printf '%s\n' "$sa_neg_ac1_before_funcs") <(p sa_neg_ac1_new_func_count="$(printf '%s\n' "$sa_neg_ac1_new_funcs" | grep -c '[^[:space:]]' || true)" assert_eq "Negative-AC-1: exactly 3 new public functions (no _ prefix)" "3" "$sa_neg_ac1_new_func_count" -# Negative AC-2: no per-key insertions in the lib +# Negative AC-2: no STRAY per-key registry insertions. The single controlled +# entry (.runtime_metrics) is set inline in the `declare -A HOOK_OWNED_FIELDS=(...)` +# line, NOT via `HOOK_OWNED_FIELDS[k]=` — so there must still be zero of those. set +e sa_neg_ac2_count="$(grep -cnE '^HOOK_OWNED_FIELDS\[' "$SA_PATH")" set -e -assert_eq "Negative-AC-2: no registry pre-population" "0" "$sa_neg_ac2_count" +assert_eq "Negative-AC-2: no stray per-key registry insertions (controlled inline entry only)" "0" "$sa_neg_ac2_count" # Negative AC-3: no scheduler-coupling identifiers in the lib file. # Pattern assembled from hex parts to prevent this script itself from matching. diff --git a/tests/test-pre-bash-contract-guard.sh b/tests/test-pre-bash-contract-guard.sh index 3802b80..11ea44d 100755 --- a/tests/test-pre-bash-contract-guard.sh +++ b/tests/test-pre-bash-contract-guard.sh @@ -253,5 +253,70 @@ assert_guard_allow \ "$CMD_E" \ "$TMP_E" +# --------------------------------------------------------------------------- +# Scenario (f): Detection 3 (proposal 4 / ST-04) -- Bash-mediated state-file +# status mutation, gated by SW_BASH_STATE_GUARD_MODE (default metric-only). +# --------------------------------------------------------------------------- +echo "" +echo "--- Scenario (f): Bash state-file status mutation (SW_BASH_STATE_GUARD_MODE) ---" +TMP_F="$(mktemp -d)" +register_cleanup "$TMP_F" +SLUG_F="state-mutate-slug" +mkdir -p "$TMP_F/.simple-workflow/backlog/briefs/active/$SLUG_F" +write_autopilot_state \ + "$TMP_F/.simple-workflow/backlog/briefs/active/$SLUG_F/autopilot-state.yaml" \ + "$SLUG_F" + +# run_guard variant that sets SW_BASH_STATE_GUARD_MODE for the hook process. +run_guard_mode() { + local mode="$1" command="$2" cwd="$3" payload so se + payload=$(jq -n --arg cmd "$command" --arg cwd "$cwd" \ + '{tool_name:"Bash", tool_input:{command:$cmd}, cwd:$cwd, session_id:"test", transcript_path:""}') + so=$(mktemp); se=$(mktemp) + set +e + printf '%s' "$payload" | env SW_BASH_STATE_GUARD_MODE="$mode" bash "$HOOK_PATH" >"$so" 2>"$se" + LAST_EXIT_CODE=$? + set -e + LAST_STDOUT=$(cat "$so"); LAST_STDERR=$(cat "$se"); rm -f "$so" "$se" +} + +CMD_F_MUTATE='yq -i ".tickets[].status = \"skipped\"" autopilot-state.yaml' + +# (f1) knob=on -> decision:block with unauthorized_state_mutate_bash + schema ref. +run_guard_mode on "$CMD_F_MUTATE" "$TMP_F" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if printf '%s' "$LAST_STDOUT" | grep -q '"decision":"block"' \ + && printf '%s' "$LAST_STDOUT" | grep -q 'unauthorized_state_mutate_bash' \ + && printf '%s' "$LAST_STDOUT" | grep -q 'docs/state-schema.md'; then + echo -e " ${GREEN}PASS${NC} (f1) knob=on: Bash status mutation blocked (unauthorized_state_mutate_bash + docs/state-schema.md ref)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} (f1) knob=on: expected decision:block. stdout: $LAST_STDOUT" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +# (f2) metric-only (the shipped default) -> NOT blocked; stderr logs would-deny. +run_guard_mode metric-only "$CMD_F_MUTATE" "$TMP_F" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if ! printf '%s' "$LAST_STDOUT" | grep -q '"decision":"block"' \ + && printf '%s' "$LAST_STDERR" | grep -q 'metric-only: would deny unauthorized_state_mutate_bash'; then + echo -e " ${GREEN}PASS${NC} (f2) metric-only (default): not blocked, logs would-deny to stderr" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} (f2) metric-only: expected allow + stderr would-deny. stdout: $LAST_STDOUT stderr: $LAST_STDERR" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +# (f3) knob=on + legitimate read-only command on a state file -> allowed (no mutation). +run_guard_mode on "grep skipped autopilot-state.yaml" "$TMP_F" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if ! printf '%s' "$LAST_STDOUT" | grep -q '"decision":"block"'; then + echo -e " ${GREEN}PASS${NC} (f3) knob=on: read-only 'grep skipped' on state file allowed (no mutation)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} (f3) knob=on: read-only command should be allowed. stdout: $LAST_STDOUT" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + echo "" print_summary diff --git a/tests/test-pre-bash-safety.sh b/tests/test-pre-bash-safety.sh index ecfc9cb..07b82c7 100755 --- a/tests/test-pre-bash-safety.sh +++ b/tests/test-pre-bash-safety.sh @@ -18,6 +18,17 @@ assert_blocked "BLOCK: rm -rf /path" \ assert_blocked "BLOCK: rm -fr /path (reversed flags)" \ "rm -fr /some/path" +# F-HOOKS-04: separated short flags `-r -f` / `-f -r` were a blind spot — the +# combined (`-rf`/`-fr`) and long-form (`-r --force`) alternatives missed them. +assert_blocked "BLOCK: rm -r -f /path (separated short flags)" \ + "rm -r -f /some/path" + +assert_blocked "BLOCK: rm -f -r /path (separated short flags, reversed)" \ + "rm -f -r /some/path" + +assert_blocked "BLOCK: rm -r -i -f /path (separated flags with intervening -i)" \ + "rm -r -i -f /some/path" + assert_blocked "BLOCK: rm -rfi /path (extra flags mixed)" \ "rm -rfi /some/path" @@ -679,6 +690,44 @@ assert_allowed "ALLOW: git push origin some--no-verify-branch (no-verify in bran echo "" +# ============================================================ +# UX-11 (proposal 5): jq-missing fail-close is knob-gated +# (SW_SAFETY_JQ_MISSING_MODE). PATH-restricted — an empty dir as PATH hides jq +# while bash is invoked by absolute path so the guard reaches its jq preflight. +# ============================================================ +_JQ_BASH="$(command -v bash)" +_JQ_NOPATH="$(mktemp -d)" +_JQ_HOOK="$HOOK_DIR/pre-bash-safety.sh" + +set +e +_jq_on_out="$(printf '{"tool_input":{"command":"echo hi"}}' | env PATH="$_JQ_NOPATH" SW_SAFETY_JQ_MISSING_MODE=on "$_JQ_BASH" "$_JQ_HOOK" 2>&1)" +_jq_on_rc=$? +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$_jq_on_rc" -eq 2 ] && printf '%s' "$_jq_on_out" | grep -qF '[SAFETY-JQ-MISSING]'; then + echo -e " ${GREEN}PASS${NC} jq-missing + knob=on: fail-closed (exit 2) with [SAFETY-JQ-MISSING] message" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} jq-missing + knob=on: expected exit 2 + message (rc=$_jq_on_rc, out=${_jq_on_out:0:80})" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +set +e +_jq_def_out="$(printf '{"tool_input":{"command":"echo hi"}}' | env PATH="$_JQ_NOPATH" "$_JQ_BASH" "$_JQ_HOOK" 2>&1)" +_jq_def_rc=$? +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$_jq_def_rc" -eq 0 ] && printf '%s' "$_jq_def_out" | grep -qF 'metric-only'; then + echo -e " ${GREEN}PASS${NC} jq-missing + default: metric-only allows (exit 0) with message" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} jq-missing + default: expected exit 0 + metric-only message (rc=$_jq_def_rc)" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi +rmdir "$_JQ_NOPATH" 2>/dev/null || true + +echo "" + # ============================================================ # Summary # ============================================================ diff --git a/tests/test-pre-edit-safety.sh b/tests/test-pre-edit-safety.sh index e360dc2..f0f6f64 100755 --- a/tests/test-pre-edit-safety.sh +++ b/tests/test-pre-edit-safety.sh @@ -205,6 +205,31 @@ else TESTS_FAILED=$((TESTS_FAILED + 1)) fi +# F-HOOKS-03: legitimate source/doc files that merely CONTAIN "credentials" or +# "secret" in the name must NOT be false-blocked — only secret-bearing +# extensions are. These two ALLOW assertions FAIL under the prior +# `credentials\b|secret\b` regex (revert guard), while `.env` / `.ssh/id_rsa` / +# `credentials.json` / `app-secret.yaml` remain blocked above. +run_edit_hook "credentials.ts" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$LAST_EXIT_CODE" -eq 0 ]; then + echo -e " ${GREEN}PASS${NC} ALLOW: credentials.ts (source file, not a secret)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} ALLOW: credentials.ts" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +run_edit_hook "secret.md" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$LAST_EXIT_CODE" -eq 0 ]; then + echo -e " ${GREEN}PASS${NC} ALLOW: secret.md (doc file, not a secret)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} ALLOW: secret.md" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + echo "" # --- PII / absolute-home-path tests --- @@ -317,4 +342,80 @@ assert_pii_allow_edit "Edge3 (edit): empty new_string allowed" \ echo "" +# UX-11 (proposal 5): jq-missing fail-close is knob-gated (SW_SAFETY_JQ_MISSING_MODE). +# PATH-restricted — an empty dir as PATH hides jq while bash is invoked by absolute +# path so the guard reaches its jq preflight. +_JQ_BASH="$(command -v bash)" +_JQ_NOPATH="$(mktemp -d)" + +set +e +_jq_on_out="$(printf '{"tool_input":{"file_path":"x"}}' | env PATH="$_JQ_NOPATH" SW_SAFETY_JQ_MISSING_MODE=on "$_JQ_BASH" "$HOOK" 2>&1)" +_jq_on_rc=$? +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$_jq_on_rc" -eq 2 ] && printf '%s' "$_jq_on_out" | grep -qF '[SAFETY-JQ-MISSING]'; then + echo -e " ${GREEN}PASS${NC} jq-missing + knob=on: fail-closed (exit 2) with [SAFETY-JQ-MISSING] message" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} jq-missing + knob=on: expected exit 2 + message (rc=$_jq_on_rc, out=${_jq_on_out:0:80})" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +set +e +_jq_def_out="$(printf '{"tool_input":{"file_path":"x"}}' | env PATH="$_JQ_NOPATH" "$_JQ_BASH" "$HOOK" 2>&1)" +_jq_def_rc=$? +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$_jq_def_rc" -eq 0 ] && printf '%s' "$_jq_def_out" | grep -qF 'metric-only'; then + echo -e " ${GREEN}PASS${NC} jq-missing + default: metric-only allows (exit 0) with message" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} jq-missing + default: expected exit 0 + metric-only message (rc=$_jq_def_rc)" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi +rmdir "$_JQ_NOPATH" 2>/dev/null || true + +echo "" + +# Proposal 4 / ST-03: HOOK_OWNED_FIELDS enforcement is gated by +# SW_STATE_FIELD_GUARD_MODE (default metric-only). With the knob ON, an Edit that +# changes the hook-owned `.runtime_metrics` field on a state file is blocked, and +# the block reason names the violated field AND references docs/state-schema.md. +# Default (metric-only) does NOT block — it logs a [STATE-FIELD-GUARD] line. +_SFG_EDIT_INPUT=$(jq -n \ + --arg fp "/tmp/sfg/autopilot-state.yaml" \ + --arg o 'runtime_metrics: []' \ + --arg n 'runtime_metrics: [{boundary: x}]' \ + '{tool_input:{file_path:$fp, old_string:$o, new_string:$n}}') + +set +e +_sfg_on_out="$(printf '%s' "$_SFG_EDIT_INPUT" | env SW_STATE_FIELD_GUARD_MODE=on bash "$HOOK" 2>&1)" +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if printf '%s' "$_sfg_on_out" | grep -q '"decision":"block"' \ + && printf '%s' "$_sfg_on_out" | grep -q 'hook_owned_field_violation' \ + && printf '%s' "$_sfg_on_out" | grep -q '\.runtime_metrics' \ + && printf '%s' "$_sfg_on_out" | grep -q 'docs/state-schema.md'; then + echo -e " ${GREEN}PASS${NC} state-field guard knob=on: .runtime_metrics Edit blocked (reason names field + docs/state-schema.md)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} state-field guard knob=on: expected decision:block with field name + schema ref. out: ${_sfg_on_out:0:140}" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +set +e +_sfg_def_out="$(printf '%s' "$_SFG_EDIT_INPUT" | env SW_STATE_FIELD_GUARD_MODE=metric-only bash "$HOOK" 2>&1)" +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if ! printf '%s' "$_sfg_def_out" | grep -q '"decision":"block"' \ + && printf '%s' "$_sfg_def_out" | grep -q '\[STATE-FIELD-GUARD\] metric-only'; then + echo -e " ${GREEN}PASS${NC} state-field guard metric-only (default): .runtime_metrics Edit not blocked, logs would-block" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} state-field guard metric-only: expected allow + [STATE-FIELD-GUARD] log. out: ${_sfg_def_out:0:140}" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +echo "" + print_summary diff --git a/tests/test-pre-write-safety.sh b/tests/test-pre-write-safety.sh index 05ffa1f..abb3ff4 100755 --- a/tests/test-pre-write-safety.sh +++ b/tests/test-pre-write-safety.sh @@ -205,6 +205,31 @@ else TESTS_FAILED=$((TESTS_FAILED + 1)) fi +# F-HOOKS-03: legitimate source/doc files that merely CONTAIN "credentials" or +# "secret" in the name must NOT be false-blocked — only secret-bearing +# extensions are. These two ALLOW assertions FAIL under the prior +# `credentials\b|secret\b` regex (revert guard), while `.env` / `.ssh/id_rsa` / +# `credentials.json` / `app-secret.yaml` remain blocked above. +run_write_hook "credentials.ts" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$LAST_EXIT_CODE" -eq 0 ]; then + echo -e " ${GREEN}PASS${NC} ALLOW: credentials.ts (source file, not a secret)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} ALLOW: credentials.ts" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +run_write_hook "secret.md" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$LAST_EXIT_CODE" -eq 0 ]; then + echo -e " ${GREEN}PASS${NC} ALLOW: secret.md (doc file, not a secret)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} ALLOW: secret.md" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + echo "" # --- PII / absolute-home-path tests --- @@ -323,4 +348,39 @@ assert_pii_allow "Neg AC6 (write): fenced /Users/runner/work/ in code block allo echo "" +# UX-11 (proposal 5): jq-missing fail-close is knob-gated (SW_SAFETY_JQ_MISSING_MODE). +# PATH-restricted — an empty dir as PATH hides jq while bash is invoked by absolute +# path so the guard reaches its jq preflight. +_JQ_BASH="$(command -v bash)" +_JQ_NOPATH="$(mktemp -d)" + +set +e +_jq_on_out="$(printf '{"tool_input":{"file_path":"x"}}' | env PATH="$_JQ_NOPATH" SW_SAFETY_JQ_MISSING_MODE=on "$_JQ_BASH" "$HOOK" 2>&1)" +_jq_on_rc=$? +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$_jq_on_rc" -eq 2 ] && printf '%s' "$_jq_on_out" | grep -qF '[SAFETY-JQ-MISSING]'; then + echo -e " ${GREEN}PASS${NC} jq-missing + knob=on: fail-closed (exit 2) with [SAFETY-JQ-MISSING] message" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} jq-missing + knob=on: expected exit 2 + message (rc=$_jq_on_rc, out=${_jq_on_out:0:80})" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +set +e +_jq_def_out="$(printf '{"tool_input":{"file_path":"x"}}' | env PATH="$_JQ_NOPATH" "$_JQ_BASH" "$HOOK" 2>&1)" +_jq_def_rc=$? +set -e +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if [ "$_jq_def_rc" -eq 0 ] && printf '%s' "$_jq_def_out" | grep -qF 'metric-only'; then + echo -e " ${GREEN}PASS${NC} jq-missing + default: metric-only allows (exit 0) with message" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} jq-missing + default: expected exit 0 + metric-only message (rc=$_jq_def_rc)" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi +rmdir "$_JQ_NOPATH" 2>/dev/null || true + +echo "" + print_summary diff --git a/tests/test-skill-contracts.sh b/tests/test-skill-contracts.sh index 0cc1be3..57781d8 100644 --- a/tests/test-skill-contracts.sh +++ b/tests/test-skill-contracts.sh @@ -2470,6 +2470,26 @@ fi echo "" +# CT-MODE-17 (README execution-chain #2 re-propagation, proposal 6): the README +# "Brief manually, then switch to autopilot" execution chain must show the +# chain=off -> set chain: on -> re-run /create-ticket brief= -> /autopilot +# re-propagation sequence (aligned with brief Step 3 + the autopilot manual-brief +# hard-stop), NOT the old /create-ticket -> /autopilot dead-end, and the stale +# `/brief mode=manual|auto` execution-chain commands must be gone. +echo "--- CT-MODE-17 ---" +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +if grep -qF 're-run /create-ticket so each ticket dir receives autopilot-policy.yaml' "$REPO_DIR/README.md" \ + && grep -qF '/create-ticket brief=.simple-workflow/backlog/briefs/active//brief.md' "$REPO_DIR/README.md" \ + && ! grep -qE '/brief mode=(manual|auto)' "$REPO_DIR/README.md"; then + echo -e " ${GREEN}PASS${NC} CT-MODE-17: README execution chain #2 documents the /create-ticket re-propagation sequence (chain=off -> chain: on -> re-run /create-ticket -> /autopilot); stale mode= execution chains removed" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} CT-MODE-17: README execution chain #2 must show the re-propagation sequence (re-run /create-ticket brief= before /autopilot) and drop the mode= execution-chain commands" + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi + +echo "" + # ============================================================================= # Category AD: /audit per-Category checklist references contract # Diff: New category. Verifies that the canonical per-Category checklist @@ -5124,6 +5144,31 @@ else fi rm -rf "$AC12_TMP" +# CT-AC-12b (proposal 4 / ST-02): product_backlog-form ticket_dir state-lie. A +# fabricated `ship: completed` on a NEVER-SCOUTED ticket whose ticket_dir still +# holds the initial `product_backlog/` form must be caught by Gate 5: the +# product_backlog -> done/ rewrite makes the -d existence check fail (no done/ +# dir) so the state-lie protection fires. Without that rewrite arm the +# product_backlog/ dir existed and the lie passed (false negative), so this +# assertion FAILs on the pre-fix code. +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +AC12B_TMP=$(mktemp -d) +mkdir -p "$AC12B_TMP/.simple-workflow/backlog/briefs/active/dummy" +mkdir -p "$AC12B_TMP/.simple-workflow/backlog/product_backlog/dummy/001-fakery" +# done/ deliberately NOT created — the ticket was never actually shipped. +AC12B_NEW=$(printf 'tickets:\n - logical_id: dummy-part-1\n ticket_dir: .simple-workflow/backlog/product_backlog/dummy/001-fakery\n status: completed\n steps:\n scout: completed\n impl: completed\n ship: completed\n') +printf '%s' "$AC12B_NEW" > "$AC12B_TMP/.simple-workflow/backlog/briefs/active/dummy/autopilot-state.yaml" +AC12B_INPUT=$(jq -n --arg fp "$AC12B_TMP/.simple-workflow/backlog/briefs/active/dummy/autopilot-state.yaml" --arg ns "$AC12B_NEW" '{tool_input:{file_path:$fp,new_string:$ns}}') +AC12B_OUT=$(cd "$AC12B_TMP" && INPUT="$AC12B_INPUT" TMUX=fake-socket INJECT_KEYS_DRY_RUN=1 SW_TEST_HARNESS=1 PATH="$AC_STUB_DIR:$PATH" bash -c "printf '%s' \"\$INPUT\" | bash \"$AC_HOOK_SAFETY\"" 2>&1 || true) +if echo "$AC12B_OUT" | grep -qE 'state-lie protection' && ! echo "$AC12B_OUT" | grep -qE '\[inject-keys\] DRY_RUN backend='; then + echo -e " ${GREEN}PASS${NC} CT-AC-12b: product_backlog-form ticket_dir state-lie caught (no inject)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} CT-AC-12b: product_backlog state-lie NOT caught. Output: $AC12B_OUT" >&2 + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi +rm -rf "$AC12B_TMP" + # CT-AC-13: dedup — fresh sentinel present -> no-op (Gate 6). When the # primary already fired and wrote .auto-compact-pending, the safety-net # must short-circuit so the user only sees ONE /compact per boundary. @@ -5165,6 +5210,29 @@ else fi rm -rf "$AC14_TMP" +# CT-AC-14b (proposal 3 / ST-11): inline-flow ship-completed payload reaches the +# dispatcher. The Gate-2 payload detector must accept the template-seeded inline +# `steps: {…, ship: completed}` shape, not only the flat `ship: completed`. +# Reverting the `completed([[:space:],}]|$)` trailing class makes the flat grep +# miss `ship: completed}`, Gate 2 short-circuits via `|| exit 0`, and the +# dispatcher is never reached — so this assertion FAILs on the pre-fix code. +TESTS_TOTAL=$((TESTS_TOTAL + 1)) +AC14B_TMP=$(mktemp -d) +mkdir -p "$AC14B_TMP/.simple-workflow/backlog/briefs/active/dummy" +mkdir -p "$AC14B_TMP/.simple-workflow/backlog/done/dummy/001-shipped" +touch "$AC14B_TMP/.simple-workflow/backlog/briefs/active/dummy/autopilot-state.yaml" +AC14B_NEW=$(printf 'tickets:\n - logical_id: dummy-part-1\n ticket_dir: .simple-workflow/backlog/done/dummy/001-shipped\n status: completed\n steps: {scout: completed, impl: completed, ship: completed}\n') +AC14B_INPUT=$(jq -n --arg fp "$AC14B_TMP/.simple-workflow/backlog/briefs/active/dummy/autopilot-state.yaml" --arg ns "$AC14B_NEW" '{tool_input:{file_path:$fp,new_string:$ns}}') +AC14B_OUT=$(cd "$AC14B_TMP" && INPUT="$AC14B_INPUT" env -u SW_AUTO_COMPACT_ON_SHIP_MODE TMUX=fake-socket INJECT_KEYS_DRY_RUN=1 SW_TEST_HARNESS=1 PATH="$AC_STUB_DIR:$PATH" bash -c "printf '%s' \"\$INPUT\" | bash \"$AC_HOOK_SAFETY\"" 2>&1 || true) +if echo "$AC14B_OUT" | grep -qE '\[inject-keys\] DRY_RUN backend='; then + echo -e " ${GREEN}PASS${NC} CT-AC-14b: safety-net — inline-flow ship-completed payload reaches dispatcher (Gate 2 flow tolerance)" + TESTS_PASSED=$((TESTS_PASSED + 1)) +else + echo -e " ${RED}FAIL${NC} CT-AC-14b: inline-flow ship-completed payload did NOT reach dispatcher. Output: $AC14B_OUT" >&2 + TESTS_FAILED=$((TESTS_FAILED + 1)) +fi +rm -rf "$AC14B_TMP" + rm -rf "$AC_STUB_DIR" # --- Shared infrastructure (CT-AC-15..22) --------------------------------- @@ -9845,6 +9913,102 @@ assert_true \ "CT-EV-PANEL-ROBUST-1 (L-ROBUSTNESS + Gate 9 R2 adversarial-key vector): ac-evaluator ($ev_protopol_acev>=1) + -hi mirror ($ev_protopol_hi>=1) + orchestration ($ev_protopol_orch>=1) + Gate 9 R2 ($ev_protopol_acqc>=1)" \ "$ev_protopol_result" +# CT-EV-PANEL-ROBUST-2 (L-ROBUSTNESS + Gate 9 R2 strictness-leniency vector, +# dogfood42-hardening): where a boundary is advertised strict/canonical/exact or +# parses a number through a lenient primitive (int()/Number()/parseInt), the panel +# lens AND the canonical Gate 9 R2 row now require probing inputs that satisfy the +# rules' letter yet exceed the advertised surface (non-ASCII/Unicode-digit numerals, +# leading-sign/whitespace forms, valid-but-non-canonical 01h / 1d0h0m0s). The shared +# token 'strictness-leniency' is wired into the ac-evaluator body + its byte-identical +# -hi twin + the orchestration panel section + Gate 9 R2. 'strictness-leniency' is +# net-new (a revert flips each to FAIL). Closes the durfmt dogfood miss (a +# 'strict-canonical' parser accepted a full-width-digit token / +5 / 01h / 1d0h0m0s). +ev_strlen_acev=$(grep -cF 'strictness-leniency' "$ACEV_EV" || true) +ev_strlen_hi=$(grep -cF 'strictness-leniency' "$ACEVHI_EV" || true) +ev_strlen_orch=$(grep -cF 'strictness-leniency' "$ORCH_EV" || true) +ev_strlen_acqc=$(grep -cF 'strictness-leniency' "$ACQC_EV" || true) +ev_strlen_result="false" +if [ "$ev_strlen_acev" -ge 1 ] && [ "$ev_strlen_hi" -ge 1 ] && [ "$ev_strlen_orch" -ge 1 ] && [ "$ev_strlen_acqc" -ge 1 ]; then ev_strlen_result="true"; fi +assert_true \ + "CT-EV-PANEL-ROBUST-2 (L-ROBUSTNESS + Gate 9 R2 strictness-leniency vector): ac-evaluator ($ev_strlen_acev>=1) + -hi mirror ($ev_strlen_hi>=1) + orchestration ($ev_strlen_orch>=1) + Gate 9 R2 ($ev_strlen_acqc>=1)" \ + "$ev_strlen_result" + +# CT-DECONTAM-1 (product-instance recidivism guard, decontamination phase). This +# guard forbids product-INSTANCE tokens from leaking into the normative evaluation +# content (the rubric, the two ac-evaluator twins, the orchestration / authoring / +# evidence references, and the planner/implementer/test-writer/ticket-evaluator +# agents). Concrete product instances (e.g. a specific color-space call, a named +# third-party library, a one-off durfmt literal) belong in transient dogfood +# transcripts, NOT baked into the generic evaluation guidance, where they bias every +# future ticket toward the colour/duration domains they came from. The denylist is +# product instances ONLY: abstract failure-class names (prototype-pollution, +# strictness-leniency) and bare colour-space / gamut / constructor vocabulary are +# DELIBERATELY excluded so this guard never collides with the CT-pinned tokens that +# CT-EV-PANEL-ROBUST-1 / -2 and their kin assert MUST be present. RED-first: on the +# contaminated tree this FAILs with a nonzero hit count and names each file+token; +# it turns green only once the later de-contamination phases strip the instances. +# NOTE: test-skill-contracts.sh itself is intentionally NOT in the normative set — +# it legitimately carries every denylist literal below. +decontam_norm_files=( + "$REPO_DIR/skills/create-ticket/references/ac-quality-criteria.md" + "$REPO_DIR/agents/ac-evaluator.md" + "$REPO_DIR/agents/ac-evaluator-hi.md" + "$REPO_DIR/skills/impl/references/ac-evaluator-orchestration.md" + "$REPO_DIR/skills/impl/references/test-authoring-guidance.md" + "$REPO_DIR/skills/impl/references/evidence-channels.md" + "$REPO_DIR/agents/planner.md" + "$REPO_DIR/agents/implementer.md" + "$REPO_DIR/agents/test-writer.md" + "$REPO_DIR/agents/ticket-evaluator.md" + "$REPO_DIR/skills/impl/references/verification-depth.md" + "$REPO_DIR/skills/impl/references/tautological-assertion-rules.md" + "$REPO_DIR/skills/impl/references/independent-oracle-harness.md" + "$REPO_DIR/skills/impl/references/accept-set-conformance-harness.md" +) +decontam_denylist=( + 'oklch(' + '1e400' + 'culori' + 'colorjs.io' + 'CSS-MINDE' + 'srgbToLinear' + '0.2126' + 'WCAG21' + 'clampChroma' + 'deltaE' + 'toGamut' + 'inGamut' + 'gamut_map' + 'parse_color' + 'U+FF11' + 'Arabic-Indic' + '01h' + '1d0h0m0s' + '__proto__' +) +decontam_total=0 +decontam_hits="" +for decontam_tok in "${decontam_denylist[@]}"; do + for decontam_f in "${decontam_norm_files[@]}"; do + [ -f "$decontam_f" ] || continue + decontam_c=$(grep -cF -- "$decontam_tok" "$decontam_f" 2>/dev/null || true) + decontam_c=${decontam_c:-0} + if [ "$decontam_c" -gt 0 ]; then + decontam_total=$((decontam_total + decontam_c)) + decontam_hits="${decontam_hits} + hit ($decontam_c) [$decontam_tok] in ${decontam_f#"$REPO_DIR"/}" + fi + done +done +decontam_result="false" +if [ "$decontam_total" -eq 0 ]; then decontam_result="true"; fi +if [ -n "$decontam_hits" ]; then + echo -e " CT-DECONTAM-1 product-instance leaks ($decontam_total total):${decontam_hits}" >&2 +fi +assert_true \ + "CT-DECONTAM-1 (product-instance recidivism guard): no product-INSTANCE tokens in the ${#decontam_norm_files[@]}-file normative set (total hits=$decontam_total, expected 0)" \ + "$decontam_result" + # CT-EV-GATE10-1 (Gate 10 peer-set uniformity, canonical + author + grader 3-file # symmetry, v8.4.0+): the new gate section exists in the canonical rubric AND the # planner self-audit names it AND the ticket-evaluator Gate-Results row names it. @@ -9884,6 +10048,48 @@ assert_true \ "CT-EV-GATE10-3 (Gate 10 kill switch wired): canonical gate ($ev_g10_3_acqc>=1) + policy doc ($ev_g10_3_apr>=1) + planner audit ($ev_g10_3_planner>=1) + policy template ($ev_g10_3_pt>=1)" \ "$ev_g10_3_result" +# CT-EV-SHARED-INPUT-XTICKET (cross-ticket shared-input-boundary signal + delegation-not-n/a +# sibling-guard clause, P-A). Both net-new tokens are HEAD=0 (RED-first). PART A pins the structured +# decomposer cross-ticket signal across producer + schema-SoT + forwarder; PART B pins the +# delegation-not-n/a clause across canonical + both byte-identical ac-evaluator twins + planner +# self-audit + ticket-evaluator grade-line. The decomposer emits the signal UNCONDITIONALLY +# (mirrors the existing unconditional peer_set: hint — decomposer.md reads no policy); downstream +# Gate 7/9 grading reuses constraints.failure_class_coverage (CT-EV-GATE9-3), so no new knob CT. +SHIN_DECOMP="$REPO_DIR/agents/decomposer.md" +SHIN_SPEC="$REPO_DIR/skills/create-ticket/references/spec-decomposer-input.md" +SHIN_SKILL="$REPO_DIR/skills/create-ticket/SKILL.md" +shin_a_decomp=$(grep -cF 'shared_input_boundary:' "$SHIN_DECOMP" || true) +shin_a_spec=$(grep -cF 'shared_input_boundary:' "$SHIN_SPEC" || true) +shin_a_skill=$(grep -cF 'shared_input_boundary:' "$SHIN_SKILL" || true) +shin_a_result="false" +if [ "$shin_a_decomp" -ge 1 ] && [ "$shin_a_spec" -ge 1 ] && [ "$shin_a_skill" -ge 1 ]; then shin_a_result="true"; fi +assert_true \ + "CT-EV-SHARED-INPUT-XTICKET PART A (decomposer cross-ticket shared_input_boundary signal: producer+schema+forwarder): decomposer ($shin_a_decomp>=1) + spec-decomposer-input ($shin_a_spec>=1) + create-ticket SKILL ($shin_a_skill>=1)" \ + "$shin_a_result" +shin_b_acqc=$(grep -ciF 'delegation is NOT an automatic n/a' "$ACQC_EV" || true) +shin_b_acev=$(grep -ciF 'delegation is NOT an automatic n/a' "$ACEV_EV" || true) +shin_b_acevhi=$(grep -ciF 'delegation is NOT an automatic n/a' "$ACEVHI_EV" || true) +shin_b_planner=$(grep -ciF 'delegation is NOT an automatic n/a' "$PLANNER_EV" || true) +shin_b_tev=$(grep -ciF 'delegation is NOT an automatic n/a' "$TEV_EV" || true) +shin_b_result="false" +if [ "$shin_b_acqc" -ge 1 ] && [ "$shin_b_acev" -ge 1 ] && [ "$shin_b_acevhi" -ge 1 ] && [ "$shin_b_planner" -ge 1 ] && [ "$shin_b_tev" -ge 1 ]; then shin_b_result="true"; fi +assert_true \ + "CT-EV-SHARED-INPUT-XTICKET PART B (delegation-not-n/a clause: canonical+both twins+planner+ticket-evaluator): acqc ($shin_b_acqc>=1) + ac-evaluator ($shin_b_acev>=1) + ac-evaluator-hi ($shin_b_acevhi>=1) + planner ($shin_b_planner>=1) + ticket-evaluator ($shin_b_tev>=1)" \ + "$shin_b_result" + +# CT-EV-SCRATCH-EVIDENCE (behavioral evidence-probe scratch carve-out clarification, +# AASC prereq). Reconciles the :199 "single exception ... computational AC" wording with +# point 5's already-directed behavioral watchdog probe (an existing inconsistency); names +# NO accept-set sweep (that is the gated AASC mechanism). Net-new token HEAD=0; byte-identical +# in both twins (CT-EV-MODEL-1 covers the identity). +scev_acev=$(grep -ciF 'behavioral evidence probe' "$ACEV_EV" || true) +scev_acevhi=$(grep -ciF 'behavioral evidence probe' "$ACEVHI_EV" || true) +scev_result="false" +if [ "$scev_acev" -ge 1 ] && [ "$scev_acevhi" -ge 1 ]; then scev_result="true"; fi +assert_true \ + "CT-EV-SCRATCH-EVIDENCE (behavioral evidence-probe carve-out clarification, both twins): ac-evaluator ($scev_acev>=1) + ac-evaluator-hi ($scev_acevhi>=1)" \ + "$scev_result" + # CT-EV-SELFDOC-1 (EC-SELFDOC channel + both failure modes, v8.4.0+). HEAD=0. ev_sd1_channel=$(grep -cF '**EC-SELFDOC**' "$ECH_EV" || true) ev_sd1_modeA=$(grep -cF 'description-vs-behavior drift' "$ECH_EV" || true) @@ -9990,6 +10196,199 @@ assert_true \ "$ev_sd6_result" +# CT-AASC-1 (accept-set conformance EXECUTED sweep + persisted observability, both twins, v8.5.0+). +# Load-bearing per charter section H: the ONLY truly additive element is the evaluator ACTUALLY +# EXECUTING the grammar-complement sweep in scratch (not more prose contract). Pins (a) the +# executed-sweep mandate (net-new phrase 'accept-set conformance' + the EXECUTE verb), and (b) the +# [ACCEPT-SET-SWEEP] marker routed into the PERSISTED report body (the M8 falsifiability fix). +# Both net-new tokens are HEAD=0 (verified RED-first: grep returns 0 files); byte-identical in both +# twins (CT-EV-MODEL-1 strip-then-diff covers the identity). A revert flips each grep to FAIL. +aasc1_acev_phrase=$(grep -ciF 'accept-set conformance' "$ACEV_EV" || true) +aasc1_hi_phrase=$(grep -ciF 'accept-set conformance' "$ACEVHI_EV" || true) +aasc1_acev_marker=$(grep -cF '[ACCEPT-SET-SWEEP]' "$ACEV_EV" || true) +aasc1_hi_marker=$(grep -cF '[ACCEPT-SET-SWEEP]' "$ACEVHI_EV" || true) +aasc1_acev_exec=$(grep -cF 'EXECUTE' "$ACEV_EV" || true) +aasc1_hi_exec=$(grep -cF 'EXECUTE' "$ACEVHI_EV" || true) +aasc1_result="false" +if [ "$aasc1_acev_phrase" -ge 1 ] && [ "$aasc1_hi_phrase" -ge 1 ] && [ "$aasc1_acev_marker" -ge 1 ] && [ "$aasc1_hi_marker" -ge 1 ] && [ "$aasc1_acev_exec" -ge 1 ] && [ "$aasc1_hi_exec" -ge 1 ]; then aasc1_result="true"; fi +assert_true \ + "CT-AASC-1 (executed accept-set sweep + observability, both twins): ac-evaluator phrase ($aasc1_acev_phrase>=1) marker ($aasc1_acev_marker>=1) EXECUTE ($aasc1_acev_exec>=1) + -hi phrase ($aasc1_hi_phrase>=1) marker ($aasc1_hi_marker>=1) EXECUTE ($aasc1_hi_exec>=1)" \ + "$aasc1_result" + +# CT-AASC-2 (four metamorphic relations + no-runnable-artifact caveat, both twins, v8.5.0+). +# Pins the executed sweep's four MR-* relations (MR-ALPHABET enumerates the Unicode decimal-digit +# complement BMP+astral naming no script) and the compiled-language Caveat arm (blocker 2: a built +# Rust/Go artifact cannot be run black-box, so a null must be distinguishable from a clean sweep). +# All five tokens HEAD=0 (verified RED-first); byte-identical in both twins. +aasc2_ok="true" +for aasc2_tok in 'MR-FINITE' 'MR-ALPHABET' 'MR-CANONICAL' 'MR-KEYFAITH' 'no-runnable-artifact'; do + aasc2_a=$(grep -cF "$aasc2_tok" "$ACEV_EV" || true) + aasc2_b=$(grep -cF "$aasc2_tok" "$ACEVHI_EV" || true) + if [ "$aasc2_a" -lt 1 ] || [ "$aasc2_b" -lt 1 ]; then aasc2_ok="false"; echo " CT-AASC-2 missing [$aasc2_tok] acev=$aasc2_a hi=$aasc2_b" >&2; fi +done +assert_true \ + "CT-AASC-2 (4 metamorphic relations + no-runnable-artifact caveat, both twins): MR-FINITE/ALPHABET/CANONICAL/KEYFAITH + caveat present in ac-evaluator AND -hi ($aasc2_ok)" \ + "$aasc2_ok" + +# CT-AASC-3 (producer-side accept-set retained-corpus obligation, both producers, v8.5.0+). +# Load-bearing per charter section H clause (iii) + blocker 9: the ac-evaluator is read-only and +# cannot write PRODUCT tests, so the fixed rejection characterization test MUST be a producer +# obligation. Pins the net-new phrase in implementer + test-writer (byte-symmetric per CLAUDE.md +# ## Modifications). HEAD=0 (verified RED-first). Reuses the existing IMPLAGENT_EV / TW_EV handles +# (declared upstream at lines 9621/9620 and 9736/9735) — NOT a fresh IMPL_AGENT_EV (Review naming-nit). +aasc3_impl=$(grep -ciF 'accept-set conformance retained corpus' "$IMPLAGENT_EV" || true) +aasc3_tw=$(grep -ciF 'accept-set conformance retained corpus' "$TW_EV" || true) +aasc3_result="false" +if [ "$aasc3_impl" -ge 1 ] && [ "$aasc3_tw" -ge 1 ]; then aasc3_result="true"; fi +assert_true \ + "CT-AASC-3 (producer-side retained-corpus obligation, both producers): implementer ($aasc3_impl>=1) + test-writer ($aasc3_tw>=1)" \ + "$aasc3_result" + +# CT-AASC-5 (accept-set-conformance-harness.md exists + wired; EC-METAMORPHIC + Grammar Card, +# v8.5.0+). Mirrors CT-EV-13 (independent-oracle-harness exists+linked). Prose-support scaffold +# (charter section H: a doc is recognition-gated), pinned so the wiring cannot silently rot. Net-new +# tokens 'accept-set-conformance-harness', 'EC-METAMORPHIC', 'Grammar Card' all HEAD=0 (verified). +AASC_DOC="$REPO_DIR/skills/impl/references/accept-set-conformance-harness.md" +aasc5_exists=0; if [ -f "$AASC_DOC" ]; then aasc5_exists=1; fi +aasc5_link_acev=$(grep -cF 'accept-set-conformance-harness' "$ACEV_EV" || true) +aasc5_link_hi=$(grep -cF 'accept-set-conformance-harness' "$ACEVHI_EV" || true) +aasc5_link_ech=$(grep -cF 'accept-set-conformance-harness' "$ECH_EV" || true) +aasc5_ecmeta=$(grep -cF 'EC-METAMORPHIC' "$ECH_EV" || true) +aasc5_grammar=$(grep -cF 'Grammar Card' "$AASC_DOC" 2>/dev/null || true) +aasc5_result="false" +if [ "$aasc5_exists" -eq 1 ] && [ "$aasc5_link_acev" -ge 1 ] && [ "$aasc5_link_hi" -ge 1 ] && [ "$aasc5_link_ech" -ge 1 ] && [ "$aasc5_ecmeta" -ge 1 ] && [ "$aasc5_grammar" -ge 1 ]; then aasc5_result="true"; fi +assert_true \ + "CT-AASC-5 (harness doc exists + wired + EC-METAMORPHIC + Grammar Card): exists ($aasc5_exists=1), linked from ac-evaluator ($aasc5_link_acev>=1) + -hi ($aasc5_link_hi>=1) + evidence-channels ($aasc5_link_ech>=1), EC-METAMORPHIC in evidence-channels ($aasc5_ecmeta>=1), Grammar Card in doc ($aasc5_grammar>=1)" \ + "$aasc5_result" + +# CT-AASC-6 (deterministic accept-set trigger: orchestrator computes + inlines triggered-on=, both twins read it; v8.5.0+). +# LOAD-BEARING per charter section H: moves the strict/canonical/lossless/limit + shared_input_boundary recognition out of +# the evaluator (dogfood46 non-uniformity) into a deterministic Step-15 computation inlined as triggered-on=. Net-new tokens +# 'triggered-on=' and (in SKILL.md) 'accept_set_conformance' are HEAD=0 (RED-first; bare 'triggered='/'accept-set' pre-exist +# from STEP 5 and are NOT what this pins). Twins byte-identical. +aasc6_skill_constraint=$(grep -cF 'accept_set_conformance' "$IMPL_EV" || true) +aasc6_skill_field=$(grep -cF 'triggered-on=' "$IMPL_EV" || true) +aasc6_skill_emit=$(grep -cF '[ACCEPT-SET-TRIGGER]' "$IMPL_EV" || true) +aasc6_acev_read=$(grep -cF 'triggered-on=' "$ACEV_EV" || true) +aasc6_hi_read=$(grep -cF 'triggered-on=' "$ACEVHI_EV" || true) +aasc6_result="false" +if [ "$aasc6_skill_constraint" -ge 1 ] && [ "$aasc6_skill_field" -ge 1 ] && [ "$aasc6_skill_emit" -ge 1 ] && [ "$aasc6_acev_read" -ge 1 ] && [ "$aasc6_hi_read" -ge 1 ]; then aasc6_result="true"; fi +assert_true "CT-AASC-6 (deterministic trigger triggered-on= computed+inlined+read): SKILL constraint ($aasc6_skill_constraint>=1) field ($aasc6_skill_field>=1) emit ($aasc6_skill_emit>=1) + ac-evaluator reads ($aasc6_acev_read>=1) + -hi reads ($aasc6_hi_read>=1)" "$aasc6_result" + +# CT-AASC-7 (black-box no-impl-peek + self-incriminating shallow-sweep record, both twins; v8.5.0+). +# LOAD-BEARING per charter section H: dogfood46 skipped astral via an implementation-peek and recorded astral=n as a neutral +# field. Pins the two GENUINELY-NEW anti-cues (the astral/no-script/property-enumeration prose already existed in STEP 5 and +# is deliberately NOT re-pinned here to avoid accretion-pinning). Both tokens HEAD=0 (RED-first). Twins byte-identical. +aasc7_ok="true" +for aasc7_tok in 'implementation-peek' 'shallow sweep'; do + aasc7_a=$(grep -cF "$aasc7_tok" "$ACEV_EV" || true) + aasc7_b=$(grep -cF "$aasc7_tok" "$ACEVHI_EV" || true) + if [ "$aasc7_a" -lt 1 ] || [ "$aasc7_b" -lt 1 ]; then aasc7_ok="false"; echo " CT-AASC-7 missing twin token [$aasc7_tok] acev=$aasc7_a hi=$aasc7_b" >&2; fi +done +assert_true "CT-AASC-7 (black-box no-impl-peek + shallow-sweep self-incrimination, both twins): implementation-peek + shallow sweep present in ac-evaluator AND -hi ($aasc7_ok)" "$aasc7_ok" + +# CT-AASC-8 (worked MR-KEYFAITH (b) shape: reflection-derived key generator + round-trip-faithfulness oracle; v8.5.0+). +# LOAD-BEARING per charter section H: a (b) dogfood self-elicited the MR-KEYFAITH / K-axis sweep but executed a HAND-PICKED +# key-literal corpus because the doc shipped a worked example ONLY for MR-ALPHABET and left MR-KEYFAITH bare prose. This pins +# that the harness doc (AASC_DOC handle) now ships a copyable MR-KEYFAITH (b) shape whose generator DERIVES the dangerous keys +# by reflection (naming NO key literal, exactly as MR-ALPHABET selects by the decimal-digit PROPERTY). Net-new token +# 'reflection-derived' is HEAD=0 (RED-first; bare 'reflection' pre-exists in the MR-KEYFAITH prose bullet and is NOT what this +# pins). Stays decontam-clean: the shape names no key-literal denylist token (CT-DECONTAM-1 must remain 0 hits). +aasc8_phrase=$(grep -cF 'reflection-derived' "$AASC_DOC" 2>/dev/null || true) +aasc8_result="false" +if [ "$aasc8_phrase" -ge 1 ]; then aasc8_result="true"; fi +assert_true \ + "CT-AASC-8 (worked MR-KEYFAITH (b) shape: reflection-derived generator + round-trip-faithfulness oracle): net-new (b)-shape phrase in harness doc ($aasc8_phrase>=1)" \ + "$aasc8_result" + +# CT-AASC-9 ((c)-parity MR-KEYFAITH CORPUS directive: derive the dangerous-key corpus BY REFLECTION, both twins; v8.5.0+). +# LOAD-BEARING per charter section H: a (b) dogfood self-elicited the MR-KEYFAITH / K-axis sweep but executed a HAND-PICKED +# key-literal corpus because the LENS shipped MR-KEYFAITH as bare/advisory prose while MR-ALPHABET (c) carried a strong +# enumerate-by-PROPERTY/names-NO-script directive (which reached astral breadth in dogfood47). This brings the MR-KEYFAITH +# CORPUS directive to (c)-parity: a MUST to DERIVE the candidate key corpus BY REFLECTION (naming NO key literal, exactly as +# MR-ALPHABET selects by the decimal-digit PROPERTY) so every reserved / accessor / colliding key is covered for ALL inputs. +# The GATING qualifier was upgraded to the shared two-tier gate in a later round (see CT-AASC-10). Net-new token +# 'DERIVE the candidate key corpus BY REFLECTION' is HEAD=0 (RED-first). Twins byte-identical. Stays decontam-clean: the +# directive names no key-literal denylist token (CT-DECONTAM-1 must remain 0 hits). +aasc9_tok='DERIVE the candidate key corpus BY REFLECTION' +aasc9_a=$(grep -cF "$aasc9_tok" "$ACEV_EV" || true) +aasc9_b=$(grep -cF "$aasc9_tok" "$ACEVHI_EV" || true) +aasc9_result="false" +if [ "$aasc9_a" -ge 1 ] && [ "$aasc9_b" -ge 1 ]; then aasc9_result="true"; fi +assert_true \ + "CT-AASC-9 ((c)-parity MR-KEYFAITH CORPUS directive: derive-by-reflection, both twins): directive present in ac-evaluator ($aasc9_a>=1) AND -hi ($aasc9_b>=1)" \ + "$aasc9_result" + +# CT-AASC-10 (MR-KEYFAITH gating proven-upgrade, both twins; (b)-finish round). +# LOAD-BEARING: the MR-KEYFAITH lens shipped its divergence as **ASSUMED, not proven** / advisory-only, a WEAKER +# floor than the other MRs. A concrete round-trip-faithfulness violation (a drop / overwrite / host-metadata mutation) +# on a lossless / strict keyed boundary is now folded into the SAME standard two-tier oracle-authoritative FAIL gating +# as MR-FINITE / MR-ALPHABET / MR-CANONICAL. This pins the folded-gate phrase present in BOTH twins AND the removal of +# the old **ASSUMED, not proven** qualifier (RED-first: HEAD ships the ASSUMED text and not the folded phrase). +aasc10_tok='SAME standard two-tier oracle-authoritative FAIL gating as the' +aasc10_a=$(grep -cF "$aasc10_tok" "$ACEV_EV" || true) +aasc10_b=$(grep -cF "$aasc10_tok" "$ACEVHI_EV" || true) +aasc10_old_a=$(grep -cF 'ASSUMED, not proven' "$ACEV_EV" || true) +aasc10_old_b=$(grep -cF 'ASSUMED, not proven' "$ACEVHI_EV" || true) +# Sibling-artifact uniformity (CLAUDE.md Modifications rule): the cross-linked EC-taxonomy +# reference both evaluators point to must NOT carry the stale ASSUMED/advisory-only posture, +# else a reader there downgrades a genuine narrow-keyed-boundary leak the twins now FAIL-gate. +aasc10_old_ech=$(grep -cF 'ASSUMED, not proven' "$ECH_EV" || true) +aasc10_result="false" +if [ "$aasc10_a" -ge 1 ] && [ "$aasc10_b" -ge 1 ] && [ "$aasc10_old_a" -eq 0 ] && [ "$aasc10_old_b" -eq 0 ] && [ "$aasc10_old_ech" -eq 0 ]; then aasc10_result="true"; fi +assert_true "CT-AASC-10 (MR-KEYFAITH gating proven-upgrade, both twins + EC ref): folded-gate phrase acev ($aasc10_a>=1) hi ($aasc10_b>=1), ASSUMED removed acev ($aasc10_old_a=0) hi ($aasc10_old_b=0) evidence-channels ($aasc10_old_ech=0)" "$aasc10_result" + +# CT-AASC-11 (round-trip oracle from input-pairs last-write-wins; harness doc). +# LOAD-BEARING: the worked MR-KEYFAITH (b) oracle must compute its expectation FROM THE INPUT PAIRS by last-write-wins, +# never by reading it back out of the builder (the dogfood-50 circular-oracle trap). This pins the net-new phrase in the +# harness doc (RED-first: HEAD=0 for the phrase). +aasc11_c=$(grep -cF 'computed from the INPUT PAIRS by last-write-wins' "$AASC_DOC" 2>/dev/null || true) +aasc11_result="false" +if [ "$aasc11_c" -ge 1 ]; then aasc11_result="true"; fi +assert_true "CT-AASC-11 (round-trip oracle from input-pairs last-write-wins): net-new phrase in harness doc ($aasc11_c>=1)" "$aasc11_result" + +# CT-AASC-12 (reflection corpus incl private/internal slots + FULL set; doc + both twins). +# LOAD-BEARING: a private-slot collision (an input key shadowing the structure's own internal storage slot) is a real +# drop / overwrite class reflection already exposes, so the reflected corpus MUST be the FULL reflected set (never sliced) +# and MUST include private / internal slot names. Pins both net-new tokens ('private / internal slot' + 'FULL reflected +# set') across the harness doc AND both twins (RED-first: HEAD=0). Stays decontam-clean: no key-literal denylist token. +aasc12_doc=$(grep -cF 'private / internal slot names' "$AASC_DOC" 2>/dev/null || true) +aasc12_acev=$(grep -cF 'private / internal slot' "$ACEV_EV" || true) +aasc12_hi=$(grep -cF 'private / internal slot' "$ACEVHI_EV" || true) +aasc12_doc_full=$(grep -cF 'FULL reflected set' "$AASC_DOC" 2>/dev/null || true) +aasc12_acev_full=$(grep -cF 'FULL reflected set' "$ACEV_EV" || true) +aasc12_hi_full=$(grep -cF 'FULL reflected set' "$ACEVHI_EV" || true) +aasc12_result="false" +if [ "$aasc12_doc" -ge 1 ] && [ "$aasc12_acev" -ge 1 ] && [ "$aasc12_hi" -ge 1 ] && [ "$aasc12_doc_full" -ge 1 ] && [ "$aasc12_acev_full" -ge 1 ] && [ "$aasc12_hi_full" -ge 1 ]; then aasc12_result="true"; fi +assert_true "CT-AASC-12 (reflection corpus incl private/internal slots + FULL set, doc+both twins): private-slot doc ($aasc12_doc) acev ($aasc12_acev) hi ($aasc12_hi); FULL-set doc ($aasc12_doc_full) acev ($aasc12_acev_full) hi ($aasc12_hi_full)" "$aasc12_result" + +# CT-AASC-13 (AASC verification hook registered: deterministic post-hoc conformance gate; v8.5.0+). +# LOAD-BEARING per charter section H: the AASC sweep + its self-incrimination rule are ALREADY normative MUST in the +# ac-evaluator lens ("an `## Accept-set sweep` line with `ran=n`, or with `ran=y astral=n`, is a NON-CONFORMANT shallow +# sweep"), yet live dogfoods leaked them run-to-run because the rule is honoured only by model RECOGNITION. +# hooks/accept-set-verify.sh reads the EMITTED `## Accept-set sweep` line from the persisted eval-round-{n}.md and applies +# the SAME rule deterministically (P1 stand-down / P2 shallow-astral / P3 sliced-corpus / P4 gating), with zero model +# recall — the only recognition-independent lever. This pins the hook wired as >=2 PostToolUse entries (Write + Edit). +# RED-first: HEAD=0 (hook absent / unwired). Behaviour is covered by tests/test-accept-set-verify.sh (not a grep CT). +aasc13_wired=$(grep -cF 'hooks/accept-set-verify.sh' "$REPO_DIR/hooks/hooks.json" || true) +aasc13_result="false" +if [ "$aasc13_wired" -ge 2 ]; then aasc13_result="true"; fi +assert_true "CT-AASC-13 (AASC verification hook registered, Write+Edit PostToolUse): hooks.json registration count ($aasc13_wired>=2)" "$aasc13_result" + +# CT-AASC-14 (accept_set_conformance kill switch documented across the 3 spawner surfaces; v8.5.0+). +# The ac-evaluator already honours an `Accept-set conformance: off` short-circuit and /impl Step 15 already reads +# `constraints.accept_set_conformance` (absent->auto), but the per-brief policy field was undocumented on every spawner +# surface, so operators could not set it — a half-wired L1/L2/L3 gap. This closes it symmetrically (CLAUDE.md ## Plans +# enumerate-every-spawner rule), mirroring CT-EV-REFUTE-2. RED-first: HEAD=0 on all three tokens. DECONTAM-clean: the +# field name is an abstract property, no product/key literal. +aasc14_pt=$(grep -ciF 'accept_set_conformance: auto' "$REPO_DIR/skills/brief/references/policy-template.md" || true) +aasc14_apr=$(grep -ciF 'constraints.accept_set_conformance' "$REPO_DIR/skills/create-ticket/references/autopilot-policy-reference.md" || true) +aasc14_orch=$(grep -ciF 'constraints.accept_set_conformance: auto|off' "$REPO_DIR/skills/impl/references/ac-evaluator-orchestration.md" || true) +aasc14_result="false" +if [ "$aasc14_pt" -ge 1 ] && [ "$aasc14_apr" -ge 1 ] && [ "$aasc14_orch" -ge 1 ]; then aasc14_result="true"; fi +assert_true "CT-AASC-14 (accept_set_conformance kill switch documented): policy-template ($aasc14_pt>=1) + policy-reference ($aasc14_apr>=1) + orchestration block ($aasc14_orch>=1)" "$aasc14_result" + + echo "" # ============================================================================= @@ -10066,6 +10465,77 @@ assert_true \ "CT-EV-MODEL-4 (criticality scalar): verification-depth 'criticality = blast_radius' ($evm4_vd>=1) + impl SKILL [CRITICALITY] stderr ($evm4_impl>=1)" \ "$evm4_result" +# ============================================================================= +# Category GEN-MODEL: Generator model policy (proposal 1'). +# sonnet_size_threshold retired; implementer = opus always; planner/decomposer = +# explicit `model: inherit`. Each assertion FAILs when the proposal-1' change is +# reverted (knob restored / inherit reverted to opus / size-routed description +# restored). +# ============================================================================= +GENM_PLANNER="$REPO_DIR/agents/planner.md" +GENM_DECOMP="$REPO_DIR/agents/decomposer.md" +GENM_IMPL_AGENT="$REPO_DIR/agents/implementer.md" + +# CT-GEN-MODEL-1: the sonnet_size_threshold knob is retired — zero reference files +# across skills/ and agents/ — AND the successor `Generator model policy` section +# exists in the canonical policy reference. +genm1_files=$( { grep -rln 'sonnet_size_threshold' "$REPO_DIR/skills" "$REPO_DIR/agents" 2>/dev/null || true; } | wc -l | tr -d ' ') +genm1_policy=$(grep -cF '## Generator model policy' "$APR_EVM" || true) +genm1_result="false" +if [ "$genm1_files" -eq 0 ] && [ "$genm1_policy" -ge 1 ]; then genm1_result="true"; fi +assert_true \ + "CT-GEN-MODEL-1 (sonnet_size_threshold retired): 0 ref files in skills/+agents/ (got $genm1_files) AND 'Generator model policy' section present ($genm1_policy>=1)" \ + "$genm1_result" + +# CT-GEN-MODEL-2: planner carries an EXPLICIT `model: inherit` (an omitted/implicit +# inherit is not accepted — self-documentation + CT verifiability). +genm2_count=$(grep -c '^model: inherit' "$GENM_PLANNER" || true) +genm2_result="false" +if [ "$genm2_count" -ge 1 ]; then genm2_result="true"; fi +assert_true \ + "CT-GEN-MODEL-2 (planner model: inherit explicit): agents/planner.md '^model: inherit' ($genm2_count>=1)" \ + "$genm2_result" + +# CT-GEN-MODEL-3: decomposer carries an EXPLICIT `model: inherit`. +genm3_count=$(grep -c '^model: inherit' "$GENM_DECOMP" || true) +genm3_result="false" +if [ "$genm3_count" -ge 1 ]; then genm3_result="true"; fi +assert_true \ + "CT-GEN-MODEL-3 (decomposer model: inherit explicit): agents/decomposer.md '^model: inherit' ($genm3_count>=1)" \ + "$genm3_result" + +# CT-GEN-MODEL-4: implementer stays pinned to `model: opus` AND no longer advertises +# the retired size-routed description ("Opus model for L/XL tickets, Sonnet for S/M"). +genm4_opus=$(grep -c '^model: opus' "$GENM_IMPL_AGENT" || true) +genm4_olddesc=$(grep -cF 'Opus model for L/XL tickets, Sonnet for S/M' "$GENM_IMPL_AGENT" || true) +genm4_result="false" +if [ "$genm4_opus" -ge 1 ] && [ "$genm4_olddesc" -eq 0 ]; then genm4_result="true"; fi +assert_true \ + "CT-GEN-MODEL-4 (implementer opus pinned, size-routed description removed): '^model: opus' ($genm4_opus>=1) AND old size-routing description absent ($genm4_olddesc==0)" \ + "$genm4_result" + +# ============================================================================= +# CT-RM-WRITERS (proposal 4 / ST-03): runtime_metrics writer enumeration sync. +# The writer list in skills/autopilot/references/state-file.md must name EVERY +# hook in the canonical `Sourced by:` header of hooks/lib/runtime-metrics.sh. A +# writer added to the lib header but not reflected in state-file.md FAILs (and +# the stale 2-of-6 enumeration this CT replaces would FAIL too). +# ============================================================================= +RMW_LIB="$REPO_DIR/hooks/lib/runtime-metrics.sh" +RMW_DOC="$REPO_DIR/skills/autopilot/references/state-file.md" +rmw_canonical=$( { sed -n '/^# Sourced by:/,/^# Public contract:/p' "$RMW_LIB" 2>/dev/null | grep -oE 'hooks/[a-z0-9-]+\.sh' || true; } | sort -u) +rmw_count=$(printf '%s\n' "$rmw_canonical" | grep -c '[^[:space:]]' || true) +rmw_missing="" +while IFS= read -r _rmw_h; do + [ -z "$_rmw_h" ] && continue + grep -qF "$_rmw_h" "$RMW_DOC" || rmw_missing="$rmw_missing $_rmw_h" +done <<< "$rmw_canonical" +rmw_result="false" +if [ "$rmw_count" -ge 6 ] && [ -z "$rmw_missing" ]; then rmw_result="true"; fi +assert_true \ + "CT-RM-WRITERS (runtime_metrics writer sync): state-file.md names all $rmw_count runtime-metrics.sh writers (missing:${rmw_missing:- none})" \ + "$rmw_result" + echo "" # --- Summary --- diff --git a/tests/test-state-parsers.sh b/tests/test-state-parsers.sh index 4fe3652..990618c 100755 --- a/tests/test-state-parsers.sh +++ b/tests/test-state-parsers.sh @@ -261,6 +261,56 @@ else fi fi +# AC-10 (proposal 3 / ST-01,ST-11): parse_active_steps is WI-3 schema-tolerant — +# it returns the SAME active (in_progress|pending) step set for the canonical-flat, +# inline-flow, and nested step shapes. The continuation driver +# (hooks/autopilot-continue.sh) relies on this to count unfinished steps and pick +# the next step across all three forms. Reverting proposal 3 removes the helper, so +# every assertion below fails (empty output). +PAS_FLAT="$SCAFFOLD_TMP/pas-flat.yaml" +PAS_FLOW="$SCAFFOLD_TMP/pas-flow.yaml" +PAS_NESTED="$SCAFFOLD_TMP/pas-nested.yaml" +cat > "$PAS_FLAT" <<'EOF' +tickets: + - logical_id: t1 + ticket_dir: 001-a + status: in_progress + steps: + create-ticket: completed + scout: in_progress + impl: pending + ship: pending +EOF +cat > "$PAS_FLOW" <<'EOF' +tickets: + - logical_id: t1 + ticket_dir: 001-a + status: in_progress + steps: {create-ticket: completed, scout: in_progress, impl: pending, ship: pending} +EOF +cat > "$PAS_NESTED" <<'EOF' +tickets: + - logical_id: t1 + ticket_dir: 001-a + status: in_progress + steps: + create-ticket: + status: completed + scout: + status: in_progress + impl: + status: pending + ship: + status: pending +EOF +PAS_EXPECT="scout:in_progress,impl:pending,ship:pending" +pas_flat="$(parse_active_steps "$PAS_FLAT" 2>/dev/null | tr '\n' ',' | sed 's/,$//')" || pas_flat="" +pas_flow="$(parse_active_steps "$PAS_FLOW" 2>/dev/null | tr '\n' ',' | sed 's/,$//')" || pas_flow="" +pas_nested="$(parse_active_steps "$PAS_NESTED" 2>/dev/null | tr '\n' ',' | sed 's/,$//')" || pas_nested="" +assert_eq "AC-10a: parse_active_steps flat form -> active steps" "$PAS_EXPECT" "$pas_flat" +assert_eq "AC-10b: parse_active_steps inline-flow form -> active steps" "$PAS_EXPECT" "$pas_flow" +assert_eq "AC-10c: parse_active_steps nested form -> active steps" "$PAS_EXPECT" "$pas_nested" + echo "" echo "===============================" echo -e "Total: $TESTS_TOTAL | ${GREEN}Passed: $TESTS_PASSED${NC} | ${RED}Failed: $TESTS_FAILED${NC}"