From a9687ba40dba84785b08e9971d070b8807e458bf Mon Sep 17 00:00:00 2001 From: Konrad Heimel Date: Wed, 19 Aug 2026 01:57:59 +0200 Subject: [PATCH 01/10] :white_check_mark: test(audit): pin the AUD2 remediations behind a PR-visible exit gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REQ-AUD2-S05-01..05. hack/audit/aud2_exitgate_test.sh asserts, per 2026-08-18 audit finding, that all four AUD2 remediations are still present: REL-01/02/07 (S01) CallExec bounds stdout at MaxResponseBytes, sets cmd.WaitDelay, and folds a bounded stderr into its error; REL-03 (S02) resolveRunFacts skips only on forge.ErrNotFound; SEC-03 (S03) hack/install.sh pins the cosign identity + OIDC issuer, byte-identically with SECURITY.md; TEST-02 (S04) isStricterInterventionEffect keeps EffectChallenge and the named unit case defends it. Three of the four have a decoy occurrence in the very same file (CallHTTP's MaxResponseBytes, loadResourceOwnerRegistry's D-130 errors.Is guard, the compare tests' EffectChallenge), so every assertion reads a function body extracted by name, with a positive control on a known-present anchor and a scoping control refusing the neighbouring decoy function. 14 mutation controls on temp copies prove each assertion goes red, and red for its own stated reason (REQ-AUD2-S05-02); the REL-03 mutant deliberately leaves the D-130 guard intact, so a file-level grep stays green on it. Wiring, in this same commit because the AUD-S18 pin exists to force exactly that: `- task: audit-aud2-exitgate-test` in Taskfile check: (19th stage) and the matching CHECK_STAGES entry in hack/audit/exitgate_test.sh. A stale pin here is the AUD-S18/RELSE-08 incident that reddened main for four merges. REQ-AUD2-S05-05: the gate runs as an early step of verify.yaml's `verify` job, which fires on pull_request — not only in release-exitgate, whose `if: github.event_name != 'pull_request'` is RELSE-08 itself. The gate asserts that placement and reds if the step is deleted or the job grows a push-only condition. What stays out of tree is recorded in the spec: making `verify` a required check is branch protection, and the `task check` stage still does not run on PRs. Also corrects verify.yaml's stale "all 14 task check stages" comment (19 now). Spec hygiene surfaced by AUD2-S03's re-review, all in the AUD2 spec: REQ-AUD2-S03-06 promotes the real-SAN fixture assertion (D-153's "assertion that would have caught the defect") from a Definition-of-done bullet to an annotated requirement pinning install_cosign_pin_test.sh §4c/§4d; the Paths owned table now records S03's SECURITY.md and CHECK_STAGES writes and S05's actual footprint; and the Problem section's line numbers are dated to b4f5054 with the SECURITY.md range corrected to that same era. --- .github/workflows/verify.yaml | 16 +- Taskfile.yml | 10 + hack/audit/aud2_exitgate_test.sh | 697 ++++++++++++++++++ hack/audit/exitgate_test.sh | 5 + .../specs/p5-aud2-audit-remediation/spec.md | 65 +- 5 files changed, 782 insertions(+), 11 deletions(-) create mode 100755 hack/audit/aud2_exitgate_test.sh diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 34c8e66..24d9309 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -37,6 +37,19 @@ jobs: # toolchain, so it runs first and fails in seconds. - name: workflow supply-chain pins (AUD-S09 — SEC-04) run: bash hack/lint/workflow_pins_test.sh + - name: AUD2 exit gate (AUD2-S05 — REQ-AUD2-S05-01..05) + # The four 2026-08-18 remediations (REL-01/02/07, REL-03, SEC-03, + # TEST-02) are still present. It runs HERE, in the job that fires on + # pull_request — NOT only in release-exitgate, whose + # `if: github.event_name != 'pull_request'` is RELSE-08, the blind spot + # that let AUD-S18's stale CHECK_STAGES pin merge green four times. + # Pure text over the source, so it needs no toolchain and fails in + # seconds. Keep it undisarmed: the gate asserts its own step carries no + # `if:`/`continue-on-error:`, so weakening this reddens it. + # These comment lines live INSIDE the step on purpose — hack/lint/ + # workflow_pins_test.sh isolates the step ABOVE up to the next `- ` + # marker and caps it at six lines. + run: bash hack/audit/aud2_exitgate_test.sh - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: stable # go.mod stays the minimum; CI runs latest stable (govulncheck needs it) @@ -140,7 +153,8 @@ jobs: - name: E9 release exit gate (RELSE-03 — REQ-E9-S13) run: bash hack/release/exitgate_test.sh # AUD-S18 (REQ-AUD-S18-01/02) — the P5-AUD exit gate: the 2026-08-06 audit's - # conditions closed, all 14 `task check` stages green, coverage at the raised + # conditions closed, all 19 `task check` stages green (the count is pinned in + # that script's CHECK_STAGES array — it read "14" until AUD2-S05), coverage at the raised # bar, the frozen JSON schemas unchanged against the v0.1.0 tag, and every # audit finding dispositioned. Invoked with NO arguments on purpose: # `--text-only` skips the toolchain layer, and the gate's own CI-wiring check diff --git a/Taskfile.yml b/Taskfile.yml index 03dd465..959c2f6 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -127,6 +127,11 @@ tasks: # Operator 2026-08-13: Dependabot is the updater; a reintroduced # renovate.json would otherwise merge green. - task: ci-audit-test + # AUD2-S05 (REQ-AUD2-S05-03): the P5-AUD2 exit gate. Pure text over the + # source, so it costs nothing here and can also run as an early step of + # the pull-request-visible `verify` job — which is the point: the four + # 2026-08-18 remediations must not be revertible behind a green PR. + - task: audit-aud2-exitgate-test ci-audit-test: desc: "E9-S04 + Dependabot-only: single CodeQL workflow; no Renovate config" @@ -267,6 +272,11 @@ tasks: cmds: - bash hack/release/install_cosign_pin_test.sh + audit-aud2-exitgate-test: + desc: "AUD2-S05 exit gate: the four 2026-08-18 remediations (REL-01/02/07, REL-03, SEC-03, TEST-02) are still present (REQ-AUD2-S05-01..05)" + cmds: + - bash hack/audit/aud2_exitgate_test.sh + audit-exitgate-test: desc: "AUD-S18 exit gate: 2026-08-06 audit conditions closed + gates green at the new bar (REQ-AUD-S18-01/02). NOT in `check` — it runs `task check` itself." cmds: diff --git a/hack/audit/aud2_exitgate_test.sh b/hack/audit/aud2_exitgate_test.sh new file mode 100755 index 0000000..845a636 --- /dev/null +++ b/hack/audit/aud2_exitgate_test.sh @@ -0,0 +1,697 @@ +#!/usr/bin/env bash +# REQ-AUD2-S05-01..05 — the P5-AUD2 exit gate for the 2026-08-18 project audit's +# "Next (risk reduction)" wave. +# +# ONE invocation asserts that all four AUD2 remediations are still present, and +# every failure names the audit finding ID it reopens: +# +# REL-01/02/07 (AUD2-S01) — internal/provider/transport.go's CallExec bounds +# child stdout at MaxResponseBytes, sets cmd.WaitDelay, and +# captures stderr into the returned error. +# REL-03 (AUD2-S02) — cmd/assent/provider_host.go's resolveRunFacts +# discriminates forge.ErrNotFound from every other FileAtRef +# error on the provider-declaration fetch. +# SEC-03 (AUD2-S03) — hack/install.sh pins the cosign signer identity and +# OIDC issuer, byte-identically with SECURITY.md. +# TEST-02 (AUD2-S04) — internal/compare's isStricterInterventionEffect +# still carries EffectChallenge, and the named unit case that +# kills the auditor's demonstrated mutant still exists. +# +# WHY THIS GATE IS TEXTUAL AND NOT A TEST RUN. Each remediation already has its +# own behavioural tests, which `task test` runs. What those tests do NOT do is +# survive their own deletion: TEST-02 is precisely the finding that a fix can be +# reverted with every wired gate still green. This gate is the anti-revert pin — +# a cheap, toolchain-free assertion over the source, which is why it can run as +# an early step of the PR-visible `verify` job (REQ-AUD2-S05-05) instead of only +# inside the push-only release-exitgate job (RELSE-08, the blind spot that hid +# AUD-S18's stale pin for four merges). +# +# SCOPING IS THE WHOLE GAME HERE. Three of the four assertions have a decoy in +# the very same file: +# * MaxResponseBytes also appears in CallHTTP, twenty lines above CallExec; +# * errors.Is(err, forge.ErrNotFound) also appears in +# loadResourceOwnerRegistry, the D-130 guard further down provider_host.go; +# * EffectChallenge also appears throughout the compare test files. +# A file-level grep for any of those stays GREEN with the remediation reverted. +# Every assertion below therefore reads a FUNCTION BODY extracted by name, with +# a positive control (the body is non-empty and contains a known-present anchor) +# and a scoping control (the body does NOT contain the neighbouring decoy +# function's header) so a broken extraction fails loudly instead of vacuously. +# +# ANTI-VACUITY DISCIPLINE (D-124, AUD-S18: this repo has a documented history of +# gates that cannot fail). Every check is a FUNCTION over file arguments, run +# twice: once against the real tree (must be GREEN) and once against a mutant +# copy carrying the very defect it exists to catch (must be RED, and red for its +# STATED reason — expect_red pins the message fragment). The mutations are +# SURGICAL: the REL-03 mutant deletes only resolveRunFacts' guard and the gate +# then asserts the decoy at loadResourceOwnerRegistry is still there, which is +# what distinguishes a scoped assertion from one that merely happens to pass. +# +# Mutants are file COPIES under mktemp -d. `git checkout --` is never used: it +# reverts to HEAD, which can be behind the working tree, so a "mutation" would +# silently be a no-op or, worse, a real edit to the tree. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +TRANSPORT="$ROOT/internal/provider/transport.go" +PROVIDER_HOST="$ROOT/cmd/assent/provider_host.go" +INSTALL="$ROOT/hack/install.sh" +SECURITY="$ROOT/SECURITY.md" +CLASSIFY="$ROOT/internal/compare/classify_intervention.go" +CLASSIFY_TEST="$ROOT/internal/compare/classify_intervention_test.go" +TASKFILE="$ROOT/Taskfile.yml" +AUD_GATE="$ROOT/hack/audit/exitgate_test.sh" +WORKFLOW="$ROOT/.github/workflows/verify.yaml" + +# This gate's own `task check` stage, and the script the stage must invoke. +STAGE="audit-aud2-exitgate-test" +SELF="hack/audit/aud2_exitgate_test.sh" +# The named unit case AUD2-S04 added; TEST-02 is closed by its existence. +CHALLENGE_TEST="TestClassifyStricterInterventionAddedChallengeEffect" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +# --------------------------------------------------------------- helpers -- + +# extract_func — the body of a top-level Go func (or method), +# from its `func` line to the first column-0 `}`. Column-0 anchoring is what +# keeps a neighbouring function out of the result. +extract_func() { + awk -v name="$2" ' + !inf && $0 ~ "^func (\\([^)]*\\) )?"name"\\(" { inf = 1; print; next } + inf && /^}/ { print; exit } + inf { print } + ' "$1" +} + +# body_of — extract with +# both controls. Returns non-zero (naming ) if the extraction produced +# nothing, lost its anchor, or swallowed the decoy function. +body_of() { + local file="$1" fn="$2" out="$3" finding="$4" anchor="$5" decoy="$6" + if [[ ! -f "$file" ]]; then + echo " ${finding}: missing ${file#"$ROOT"/}" >&2 + return 1 + fi + extract_func "$file" "$fn" >"$out" + if [[ ! -s "$out" ]]; then + echo " ${finding}: func ${fn} was not found in ${file#"$ROOT"/} — it was renamed or deleted, and every assertion over its body would be vacuous" >&2 + return 1 + fi + if ! grep -Fq -- "$anchor" "$out"; then + echo " ${finding}: the extracted ${fn} body does not contain the known-present anchor '${anchor}' — the extraction is wrong, so the assertions below would be vacuous" >&2 + return 1 + fi + if [[ -n "$decoy" ]] && grep -Fq -- "$decoy" "$out"; then + echo " ${finding}: the extracted ${fn} body ALSO contains '${decoy}' — the extraction ran past the end of the function and into a neighbour that carries a decoy occurrence, so a same-file grep is what is really being graded" >&2 + return 1 + fi + return 0 +} + +# rhs_of — the right-hand side of the first assignment to +# an lvalue, with any leading `&` stripped. Empty when there is no assignment. +rhs_of() { + sed -nE "s/^[[:space:]]*$2[[:space:]]*=[[:space:]]*&?(.+)$/\1/p" "$1" | head -1 +} + +# delete_first — delete only the FIRST line matching the pattern, +# in place. Surgical by construction: a later decoy occurrence survives, which +# is what makes the mutation a real test of the assertion's scoping. +delete_first() { + local file="$1" re="$2" + awk -v re="$re" 'done != 1 && $0 ~ re { done = 1; next } { print }' "$file" >"$file.mut" + mv "$file.mut" "$file" +} + +# mutate — sed in place, then prove the mutation +# actually landed. A mutant identical to the clean input proves nothing. +mutate() { + local file="$1" program="$2" witness="$3" before after + before="$(cat "$file")" + sed "$program" "$file" >"$file.mut" + mv "$file.mut" "$file" + after="$(cat "$file")" + [[ "$before" != "$after" ]] || + fail "mutation harness: sed '$program' did not change ${file#"$WORK"/} — the mutant equals the clean input, so any 'the check fires' conclusion would be false" + grep -Fq -- "$witness" "$file" || + fail "mutation harness: expected '$witness' in ${file#"$WORK"/} after mutation, but it is absent" +} + +# assert_assignment_gone — the mutation really +# removed the ASSIGNMENT, not some prose mention of it. Deleting a doc comment +# also "changes the file", which is why assert_changed alone is not enough. +assert_assignment_gone() { + ! grep -Eq "^[[:space:]]*$2[[:space:]]*=" "$1" || + fail "mutation harness ($3): ${1#"$WORK"/} still assigns $2 — the deletion hit a different line (a comment, most likely) and the control would grade the wrong thing" +} + +# assert_changed — the mutant really differs from its source. +assert_changed() { + ! diff -q "$1" "$2" >/dev/null 2>&1 || + fail "mutation harness: ${2#"$WORK"/} is byte-identical to its source — the mutation did not land" +} + +expect_green() { #