From 072cda2fd6a5fd9abd93460f0d64c02bd8e90136 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 26 Aug 2026 22:34:21 +0000 Subject: [PATCH] Harden release evidence and operator workflows --- .github/workflows/benchmark-calibration.yml | 430 +++++++++++ .github/workflows/release.yml | 473 ++++++++++-- Cargo.lock | 439 ++++++++++- Cargo.toml | 3 +- ROADMAP.md | 29 +- bench/README.md | 185 ++++- bench/baseline.json | 25 +- bench/evaluator-contract-sources.json | 7 + bench/package.json | 3 + bench/review-contract-sources.json | 1 + bench/src/cohort-run.ts | 286 +++++++ bench/src/cohort.test.ts | 254 +++++++ bench/src/cohort.ts | 375 +++++++++ bench/src/compare-baseline.test.ts | 563 +++++++++++++- bench/src/compare-baseline.ts | 635 ++++++++++++++-- bench/src/generation-evidence.test.ts | 144 ++++ bench/src/generation-evidence.ts | 241 ++++++ bench/src/live.test.ts | 25 + bench/src/live.ts | 196 +++-- bench/src/livemodels.test.ts | 192 ++++- bench/src/request-window.test.ts | 35 + bench/src/request-window.ts | 18 +- src/alerts.rs | 798 ++++++++++++++++++++ src/cli.rs | 44 ++ src/config.rs | 25 + src/credentials.rs | 299 ++++++++ src/lib.rs | 1 + src/llm.rs | 105 ++- src/login.rs | 146 +++- src/main.rs | 12 +- src/output.rs | 40 +- src/progress.rs | 4 + src/review.rs | 32 +- tests/e2e.rs | 135 +++- 34 files changed, 5770 insertions(+), 430 deletions(-) create mode 100644 .github/workflows/benchmark-calibration.yml create mode 100644 bench/src/cohort-run.ts create mode 100644 bench/src/cohort.test.ts create mode 100644 bench/src/cohort.ts create mode 100644 bench/src/generation-evidence.test.ts create mode 100644 bench/src/generation-evidence.ts create mode 100644 src/alerts.rs diff --git a/.github/workflows/benchmark-calibration.yml b/.github/workflows/benchmark-calibration.yml new file mode 100644 index 0000000..b943856 --- /dev/null +++ b/.github/workflows/benchmark-calibration.yml @@ -0,0 +1,430 @@ +name: Benchmark calibration + +on: + workflow_dispatch: {} + +concurrency: + group: benchmark-calibration-${{ github.sha }} + cancel-in-progress: false + +jobs: + validate: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Reserve the current main calibration source + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + test "${GITHUB_REF}" = "refs/heads/main" || { + echo "::error::Benchmark calibration runs only from main." + exit 1 + } + test "${GITHUB_RUN_ATTEMPT}" = "1" || { + echo "::error::Calibration retries are not authoritative. Fix the source and start a new run." + exit 1 + } + main_sha="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq .object.sha)" + test "${GITHUB_SHA}" = "$main_sha" || { + echo "::error::Calibration source is not the current main commit." + exit 1 + } + gh api --method POST "repos/${GITHUB_REPOSITORY}/git/refs" \ + -f ref="refs/tags/postil-calibration-${GITHUB_SHA}" \ + -f sha="${GITHUB_SHA}" \ + --silent + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + - name: Verify embedded model admission + working-directory: bench + run: bun run verify-admission --allow-provisional + - name: Require an unpopulated Luna baseline + run: >- + jq -e '.profiles["openai/gpt-5.6-luna"].populated == false' + bench/baseline.json >/dev/null + + prepare: + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 270 + permissions: + contents: read + id-token: write + attestations: write + artifact-metadata: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Require a working OpenRouter credential + env: + COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + set -euo pipefail + if [[ -z "${COMPLETION_API_KEY}" ]]; then + echo "::error::OPENROUTER_API_KEY is not set." + exit 1 + fi + status="$(curl --fail-with-body --silent --show-error \ + --output /tmp/openrouter-key.json --write-out '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${COMPLETION_API_KEY}" \ + https://openrouter.ai/api/v1/key)" || { + echo "::error::OPENROUTER_API_KEY was rejected by OpenRouter (HTTP ${status:-none})." + exit 1 + } + jq -e '.data.limit_remaining == null or .data.limit_remaining >= 1' \ + /tmp/openrouter-key.json >/dev/null || { + echo "::error::OPENROUTER_API_KEY has insufficient calibration credit." + exit 1 + } + - run: cargo build --quiet --release + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + - name: Predeclare the calibration cohort + working-directory: bench + run: >- + bun run bench:cohort-create -- + --purpose calibration + --binary "${{ github.workspace }}/target/release/postil" + --screen-profile ../provisional-models.json + --run-prefix "calibration-${{ github.run_id }}" + --out "${{ runner.temp }}/benchmark-calibration-cohort.json" + - name: Attest the calibration binary and cohort + id: attest-prepared + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: | + ${{ github.workspace }}/target/release/postil + ${{ runner.temp }}/benchmark-calibration-cohort.json + - name: Stage prepared calibration evidence + run: | + set -euo pipefail + evidence="${RUNNER_TEMP}/benchmark-calibration-prepared" + mkdir -p "$evidence" + cp target/release/postil "$evidence/postil" + cp "${RUNNER_TEMP}/benchmark-calibration-cohort.json" "$evidence/cohort.json" + cp "${{ steps.attest-prepared.outputs.bundle-path }}" "$evidence/prepared.attestation.json" + - name: Upload prepared calibration evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-calibration-prepared-${{ github.run_id }} + path: ${{ runner.temp }}/benchmark-calibration-prepared + if-no-files-found: error + retention-days: 90 + + sample: + needs: [validate, prepare] + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + id-token: write + attestations: write + artifact-metadata: write + strategy: + fail-fast: false + max-parallel: 1 + matrix: + sample: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Download prepared calibration evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmark-calibration-prepared-${{ github.run_id }} + path: ${{ runner.temp }}/prepared + - name: Verify prepared calibration provenance + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + chmod 0755 "${RUNNER_TEMP}/prepared/postil" + for subject in postil cohort.json; do + gh attestation verify "${RUNNER_TEMP}/prepared/${subject}" \ + --bundle "${RUNNER_TEMP}/prepared/prepared.attestation.json" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref refs/heads/main \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json >/dev/null + done + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + - name: Resolve canonical sample slot + run: echo "CALIBRATION_SLOT=$(printf '%02d' '${{ matrix.sample }}')" >> "$GITHUB_ENV" + - name: Reserve benchmark sample ${{ matrix.sample }} + working-directory: bench + run: >- + bun run bench:cohort-run -- + --mode reserve + --manifest "${{ runner.temp }}/prepared/cohort.json" + --slot "${{ matrix.sample }}" + --binary "${{ runner.temp }}/prepared/postil" + --screen-profile ../provisional-models.json + - name: Attest benchmark sample reservation + id: attest-reservation + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ runner.temp }}/prepared/slots/${{ env.CALIBRATION_SLOT }}/receipt.json + - name: Preserve reservation attestation + run: | + cp "${{ steps.attest-reservation.outputs.bundle-path }}" \ + "${RUNNER_TEMP}/reservation.attestation.json" + cp "${RUNNER_TEMP}/prepared/slots/${CALIBRATION_SLOT}/receipt.json" \ + "${RUNNER_TEMP}/reservation.receipt.json" + - name: Execute benchmark sample ${{ matrix.sample }} + working-directory: bench + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + POSTIL_BIN: ${{ runner.temp }}/prepared/postil + run: >- + bun run bench:cohort-run -- + --mode execute + --manifest "${{ runner.temp }}/prepared/cohort.json" + --slot "${{ matrix.sample }}" + --binary "${{ runner.temp }}/prepared/postil" + --screen-profile ../provisional-models.json + - name: Attest benchmark sample result + id: attest-result + if: success() + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: | + ${{ runner.temp }}/prepared/slots/${{ env.CALIBRATION_SLOT }}/report.json + ${{ runner.temp }}/prepared/slots/${{ env.CALIBRATION_SLOT }}/receipt.json + - name: Stage benchmark sample evidence + if: always() + run: | + set -euo pipefail + slot="$(printf '%02d' '${{ matrix.sample }}')" + source="${RUNNER_TEMP}/prepared/slots/${slot}" + evidence="${RUNNER_TEMP}/benchmark-calibration-sample-${slot}" + mkdir -p "$evidence" + cp "$source/receipt.json" "$evidence/receipt.json" + cp "${RUNNER_TEMP}/reservation.receipt.json" "$evidence/reservation.receipt.json" + cp "${RUNNER_TEMP}/reservation.attestation.json" "$evidence/reservation.attestation.json" + if [[ -f "$source/report.json" ]]; then cp "$source/report.json" "$evidence/report.json"; fi + if [[ -n "${{ steps.attest-result.outputs.bundle-path }}" && -f "${{ steps.attest-result.outputs.bundle-path }}" ]]; then + cp "${{ steps.attest-result.outputs.bundle-path }}" "$evidence/result.attestation.json" + fi + - name: Upload benchmark sample evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-calibration-sample-${{ github.run_id }}-${{ matrix.sample }} + path: ${{ runner.temp }}/benchmark-calibration-sample-${{ env.CALIBRATION_SLOT }} + if-no-files-found: error + retention-days: 90 + + record: + needs: [validate, prepare, sample] + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + id-token: write + attestations: write + artifact-metadata: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Download prepared calibration evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmark-calibration-prepared-${{ github.run_id }} + path: ${{ runner.temp }}/prepared + - name: Download calibration sample evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: benchmark-calibration-sample-${{ github.run_id }}-* + path: ${{ runner.temp }}/samples + merge-multiple: false + - name: Normalize and verify all calibration evidence + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + chmod 0755 "${RUNNER_TEMP}/prepared/postil" + for subject in postil cohort.json; do + gh attestation verify "${RUNNER_TEMP}/prepared/${subject}" \ + --bundle "${RUNNER_TEMP}/prepared/prepared.attestation.json" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref refs/heads/main \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json >/dev/null + done + mkdir -p "${RUNNER_TEMP}/verified-slots" + for sample in 1 2 3 4 5 6 7 8 9 10; do + slot="$(printf '%02d' "$sample")" + destination="${RUNNER_TEMP}/verified-slots/${slot}" + source="${RUNNER_TEMP}/samples/benchmark-calibration-sample-${GITHUB_RUN_ID}-${sample}" + expected_inventory=$'receipt.json\nreport.json\nreservation.attestation.json\nreservation.receipt.json\nresult.attestation.json' + actual_inventory="$(find "$source" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort)" + test "$actual_inventory" = "$expected_inventory" || { + echo "::error::Calibration slot ${slot} has an unexpected artifact inventory." + exit 1 + } + mkdir -p "$destination" + for name in report.json receipt.json reservation.receipt.json reservation.attestation.json result.attestation.json; do + test -f "$source/$name" && test ! -L "$source/$name" && test -s "$source/$name" || { + echo "::error::Missing ${name} for calibration slot ${slot}." + exit 1 + } + cp "$source/$name" "$destination/$name" + done + gh attestation verify "$destination/reservation.receipt.json" \ + --bundle "$destination/reservation.attestation.json" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref refs/heads/main \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json >/dev/null + jq -e -s ' + .[0].state == "running" and + .[1].state == "completed" and + .[0].manifestSha256 == .[1].manifestSha256 and + .[0].cohortId == .[1].cohortId and + .[0].purpose == .[1].purpose and + .[0].slot == .[1].slot and + .[0].nonce == .[1].nonce and + .[0].runId == .[1].runId and + .[0].startedAt == .[1].startedAt + ' "$destination/reservation.receipt.json" "$destination/receipt.json" >/dev/null || { + echo "::error::Calibration slot ${slot} terminal receipt is not bound to its reservation." + exit 1 + } + for subject in report.json receipt.json; do + gh attestation verify "$destination/${subject}" \ + --bundle "$destination/result.attestation.json" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref refs/heads/main \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json >/dev/null + done + done + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + - name: Verify independent calibration generations + working-directory: bench + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + bun run bench:verify-generations -- \ + --result "${RUNNER_TEMP}/verified-slots/01/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/01/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/02/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/02/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/03/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/03/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/04/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/04/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/05/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/05/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/06/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/06/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/07/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/07/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/08/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/08/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/09/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/09/receipt.json" \ + --result "${RUNNER_TEMP}/verified-slots/10/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/10/receipt.json" + - name: Record the attested baseline + working-directory: bench + run: | + set -euo pipefail + bun run bench:compare -- \ + --binary "${RUNNER_TEMP}/prepared/postil" \ + --screen-profile ../provisional-models.json \ + --cohort-manifest "${RUNNER_TEMP}/prepared/cohort.json" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-01" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-02" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-03" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-04" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-05" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-06" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-07" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-08" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-09" \ + --expected-run-id "calibration-${GITHUB_RUN_ID}-10" \ + --result "${RUNNER_TEMP}/verified-slots/01/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/02/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/03/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/04/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/05/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/06/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/07/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/08/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/09/report.json" \ + --result "${RUNNER_TEMP}/verified-slots/10/report.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/01/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/02/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/03/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/04/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/05/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/06/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/07/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/08/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/09/receipt.json" \ + --receipt "${RUNNER_TEMP}/verified-slots/10/receipt.json" \ + --record + - name: Attest the populated baseline + id: attest-baseline + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ github.workspace }}/bench/baseline.json + - name: Stage populated baseline evidence + run: | + mkdir -p "${RUNNER_TEMP}/populated-baseline" + cp bench/baseline.json "${RUNNER_TEMP}/populated-baseline/baseline.json" + cp "${{ steps.attest-baseline.outputs.bundle-path }}" \ + "${RUNNER_TEMP}/populated-baseline/baseline.attestation.json" + - name: Upload populated baseline + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-calibration-baseline-${{ github.run_id }} + path: ${{ runner.temp }}/populated-baseline + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b2beec..a84d3b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,14 +3,48 @@ name: Release on: push: tags: ["v*"] - workflow_dispatch: {} + +concurrency: + group: release-${{ github.ref_name }} + cancel-in-progress: false jobs: validate-tag: runs-on: ubuntu-latest permissions: contents: read + actions: read steps: + - name: Require the unique first release run for this tag + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if [[ "${GITHUB_RUN_ATTEMPT}" != "1" ]]; then + echo "::error::Release workflow retries are not authoritative. Create a new version instead." + exit 1 + fi + gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs" \ + -f event=push \ + -f branch="${GITHUB_REF_NAME}" \ + -f per_page=100 \ + | jq -e \ + --arg id "${GITHUB_RUN_ID}" \ + --arg tag "${GITHUB_REF_NAME}" \ + '[.workflow_runs[] | select( + .event == "push" and + .head_branch == $tag + )] as $runs | + ($runs | length) == 1 and (($runs[0].id | tostring) == $id)' \ + >/dev/null || { + echo "::error::This version tag already has another release run." + exit 1 + } + if gh release view "${GITHUB_REF_NAME}" >/dev/null 2>&1; then + echo "::error::A GitHub release already exists for this version tag." + exit 1 + fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 @@ -31,6 +65,42 @@ jobs: - name: Verify embedded model admission working-directory: bench run: bun run verify-admission --allow-provisional + - name: Verify the attested Luna calibration baseline + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + profile="openai/gpt-5.6-luna" + test -s bench/baseline.attestation.json || { + echo "::error::The populated Luna baseline has no committed calibration attestation." + exit 1 + } + source_sha="$(jq -er --arg profile "$profile" \ + '.profiles[$profile] | select(.populated == true) | .calibration.sourceSha' \ + bench/baseline.json)" + [[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || { + echo "::error::The Luna baseline has an invalid calibration source digest." + exit 1 + } + registry_sha="$(gh api \ + "repos/${GITHUB_REPOSITORY}/git/ref/tags/postil-calibration-${source_sha}" \ + --jq .object.sha)" + test "$registry_sha" = "$source_sha" || { + echo "::error::The Luna baseline source is not protected by the calibration registry." + exit 1 + } + gh attestation verify bench/baseline.json \ + --bundle bench/baseline.attestation.json \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ + --signer-digest "$source_sha" \ + --source-digest "$source_sha" \ + --source-ref refs/heads/main \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json >/dev/null - name: Match release tag to package version env: RELEASE_TAG: ${{ github.ref_name }} @@ -41,12 +111,15 @@ jobs: exit 1 } - bench-live: + bench-live-prepare: needs: validate-tag runs-on: ubuntu-latest timeout-minutes: 270 permissions: contents: read + id-token: write + attestations: write + artifact-metadata: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable @@ -83,12 +156,10 @@ jobs: fi if jq -e '.data.limit_remaining != null and .data.limit_remaining < 1' \ /tmp/openrouter-key.json >/dev/null; then - remaining="$(jq -r '.data.limit_remaining' /tmp/openrouter-key.json)" - echo "::error::OPENROUTER_API_KEY has ${remaining} credit remaining, below the release benchmark reserve." + echo "::error::OPENROUTER_API_KEY has insufficient credit for the release benchmark reserve." exit 1 fi - remaining="$(jq -r '.data.limit_remaining // "unlimited"' /tmp/openrouter-key.json)" - echo "OpenRouter credential accepted (remaining: ${remaining})." + echo "OpenRouter credential accepted." # Plain release profile: the same build the release job ships, so the # gate measures the binary users will actually get. @@ -96,7 +167,7 @@ jobs: - name: Preserve benchmarked Linux binary uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: benchmarked-x86_64-unknown-linux-gnu + name: benchmarked-x86_64-unknown-linux-gnu-${{ github.run_attempt }} path: target/release/postil if-no-files-found: error retention-days: 1 @@ -104,6 +175,38 @@ jobs: working-directory: bench run: bun install --frozen-lockfile + - name: Predeclare the release benchmark cohort + working-directory: bench + run: >- + bun run bench:cohort-create -- + --purpose release + --binary "${{ github.workspace }}/target/release/postil" + --screen-profile ../provisional-models.json + --run-prefix "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}" + --out "${{ runner.temp }}/bench-live-cohort.json" + + - name: Attest the release benchmark cohort + id: attest-cohort + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ runner.temp }}/bench-live-cohort.json + + - name: Stage the release benchmark cohort evidence + run: | + set -euo pipefail + evidence_dir="${RUNNER_TEMP}/bench-live-cohort-evidence" + mkdir -p "$evidence_dir" + cp "${RUNNER_TEMP}/bench-live-cohort.json" "$evidence_dir/bench-live-cohort.json" + cp "${{ steps.attest-cohort.outputs.bundle-path }}" "$evidence_dir/bench-live-cohort.attestation.json" + + - name: Upload the release benchmark cohort and attestation + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-live-cohort-${{ github.run_attempt }} + path: ${{ runner.temp }}/bench-live-cohort-evidence + if-no-files-found: error + retention-days: 30 + - name: Run the embedded hosted scorer gate working-directory: bench env: @@ -121,83 +224,335 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: scorer-eval-report + name: scorer-eval-report-${{ github.run_attempt }} path: | ${{ runner.temp }}/scorer-eval-report.json ${{ runner.temp }}/scorer-eval-report.json.partial if-no-files-found: warn retention-days: 30 - - name: Run diff-file live benchmark sample 1 - id: live-sample-1 - continue-on-error: true - working-directory: bench + bench-live-sample: + needs: [validate-tag, bench-live-prepare] + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + id-token: write + attestations: write + artifact-metadata: write + strategy: + fail-fast: false + max-parallel: 1 + matrix: + sample: [1, 2, 3, 4, 5] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Download benchmarked Linux binary + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmarked-x86_64-unknown-linux-gnu-${{ github.run_attempt }} + path: target/release + - name: Download predeclared release benchmark cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bench-live-cohort-${{ github.run_attempt }} + path: ${{ runner.temp }}/bench-live-cohort-download + - name: Normalize and verify the predeclared release benchmark cohort env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - POSTIL_BIN: ${{ github.workspace }}/target/release/postil - run: >- - bun run bench:live -- --screen-profile ../provisional-models.json - --run-id "release-${{ github.ref_name }}-sample-1" - --json-out "${{ runner.temp }}/bench-live-report-1.json" - - - name: Run diff-file live benchmark sample 2 - id: live-sample-2 - continue-on-error: true + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + source="${RUNNER_TEMP}/bench-live-cohort-download" + expected_inventory=$'bench-live-cohort.attestation.json\nbench-live-cohort.json' + actual_inventory="$(find "$source" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort)" + if [[ "$actual_inventory" != "$expected_inventory" ]] || + [[ ! -f "$source/bench-live-cohort.json" ]] || + [[ -L "$source/bench-live-cohort.json" ]] || + [[ ! -f "$source/bench-live-cohort.attestation.json" ]] || + [[ -L "$source/bench-live-cohort.attestation.json" ]]; then + echo "::error::Expected exactly one release cohort manifest and attestation bundle." + exit 1 + fi + cp "$source/bench-live-cohort.json" "${RUNNER_TEMP}/bench-live-cohort.json" + cp "$source/bench-live-cohort.attestation.json" "${RUNNER_TEMP}/bench-live-cohort.attestation.json" + echo "COHORT_MANIFEST=${RUNNER_TEMP}/bench-live-cohort.json" >> "$GITHUB_ENV" + echo "COHORT_BUNDLE=${RUNNER_TEMP}/bench-live-cohort.attestation.json" >> "$GITHUB_ENV" + gh attestation verify "${RUNNER_TEMP}/bench-live-cohort.json" \ + --bundle "${RUNNER_TEMP}/bench-live-cohort.attestation.json" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/release.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref "${GITHUB_REF}" \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json > /dev/null + - run: chmod 0755 target/release/postil + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + - name: Resolve canonical release sample slot + run: echo "RELEASE_SLOT=$(printf '%02d' '${{ matrix.sample }}')" >> "$GITHUB_ENV" + - name: Reserve diff-file live benchmark sample ${{ matrix.sample }} working-directory: bench - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - POSTIL_BIN: ${{ github.workspace }}/target/release/postil run: >- - bun run bench:live -- --screen-profile ../provisional-models.json - --run-id "release-${{ github.ref_name }}-sample-2" - --json-out "${{ runner.temp }}/bench-live-report-2.json" - - - name: Run diff-file live benchmark sample 3 - id: live-sample-3 - continue-on-error: true + bun run bench:cohort-run -- + --mode reserve + --manifest "${{ runner.temp }}/bench-live-cohort.json" + --slot "${{ matrix.sample }}" + --binary "${{ github.workspace }}/target/release/postil" + --screen-profile ../provisional-models.json + - name: Attest benchmark sample reservation + id: attest-reservation + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ runner.temp }}/slots/${{ env.RELEASE_SLOT }}/receipt.json + - name: Preserve benchmark sample reservation + run: | + cp "${{ runner.temp }}/slots/${{ env.RELEASE_SLOT }}/receipt.json" \ + "${{ runner.temp }}/reservation.receipt.json" + cp "${{ steps.attest-reservation.outputs.bundle-path }}" \ + "${{ runner.temp }}/reservation.attestation.json" + - name: Run diff-file live benchmark sample ${{ matrix.sample }} working-directory: bench env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} POSTIL_BIN: ${{ github.workspace }}/target/release/postil run: >- - bun run bench:live -- --screen-profile ../provisional-models.json - --run-id "release-${{ github.ref_name }}-sample-3" - --json-out "${{ runner.temp }}/bench-live-report-3.json" - - - name: Upload the diff-file live reports + bun run bench:cohort-run -- + --mode execute + --manifest "${{ runner.temp }}/bench-live-cohort.json" + --slot "${{ matrix.sample }}" + --binary "${{ github.workspace }}/target/release/postil" + --screen-profile ../provisional-models.json + - name: Verify successful benchmark sample evidence + if: success() + run: | + set -euo pipefail + slot_dir="${RUNNER_TEMP}/slots/$(printf '%02d' '${{ matrix.sample }}')" + test -s "$slot_dir/report.json" + test -s "$slot_dir/receipt.json" + jq -e '.state == "completed"' "$slot_dir/receipt.json" >/dev/null + - name: Attest benchmark sample report and receipt + id: attest-sample + if: success() + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: | + ${{ runner.temp }}/slots/${{ env.RELEASE_SLOT }}/report.json + ${{ runner.temp }}/slots/${{ env.RELEASE_SLOT }}/receipt.json + - name: Stage benchmark sample evidence + if: always() + run: | + set -euo pipefail + slot="${RELEASE_SLOT}" + slot_dir="${RUNNER_TEMP}/slots/${slot}" + evidence_dir="${RUNNER_TEMP}/bench-live-sample-${{ github.run_attempt }}-${{ matrix.sample }}" + mkdir -p "${evidence_dir}/slots/${slot}" + if [[ -f "${slot_dir}/report.json" ]]; then cp "${slot_dir}/report.json" "${evidence_dir}/slots/${slot}/report.json"; fi + if [[ -f "${slot_dir}/receipt.json" ]]; then cp "${slot_dir}/receipt.json" "${evidence_dir}/slots/${slot}/receipt.json"; fi + cp "${RUNNER_TEMP}/reservation.receipt.json" "${evidence_dir}/slots/${slot}/reservation.receipt.json" + cp "${RUNNER_TEMP}/reservation.attestation.json" "${evidence_dir}/slots/${slot}/reservation.attestation.json" + if [[ -n "${{ steps.attest-sample.outputs.bundle-path }}" && -f "${{ steps.attest-sample.outputs.bundle-path }}" ]]; then + cp "${{ steps.attest-sample.outputs.bundle-path }}" "${evidence_dir}/slots/${slot}/attestation.bundle.json" + fi + - name: Upload diff-file live benchmark sample ${{ matrix.sample }} if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: bench-live-reports - path: | - ${{ runner.temp }}/bench-live-report-1.json - ${{ runner.temp }}/bench-live-report-2.json - ${{ runner.temp }}/bench-live-report-3.json + name: bench-live-sample-${{ github.run_attempt }}-${{ matrix.sample }} + path: ${{ runner.temp }}/bench-live-sample-${{ github.run_attempt }}-${{ matrix.sample }} if-no-files-found: warn retention-days: 30 + bench-live: + if: always() + needs: [bench-live-prepare, bench-live-sample] + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Download benchmarked Linux binary + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmarked-x86_64-unknown-linux-gnu-${{ github.run_attempt }} + path: target/release + - run: chmod 0755 target/release/postil + - name: Download diff-file live reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: bench-live-sample-${{ github.run_attempt }}-* + path: ${{ runner.temp }}/bench-live-reports-download + merge-multiple: false + - name: Download predeclared release benchmark cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bench-live-cohort-${{ github.run_attempt }} + path: ${{ runner.temp }}/bench-live-cohort-download + - name: Normalize release benchmark evidence paths + run: | + set -euo pipefail + cohort_source="${RUNNER_TEMP}/bench-live-cohort-download" + cohort_expected=$'bench-live-cohort.attestation.json\nbench-live-cohort.json' + cohort_actual="$(find "$cohort_source" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort)" + if [[ "$cohort_actual" != "$cohort_expected" ]] || + [[ ! -f "$cohort_source/bench-live-cohort.json" ]] || + [[ -L "$cohort_source/bench-live-cohort.json" ]] || + [[ ! -f "$cohort_source/bench-live-cohort.attestation.json" ]] || + [[ -L "$cohort_source/bench-live-cohort.attestation.json" ]]; then + echo "::error::Expected exactly one release cohort manifest and attestation bundle." + exit 1 + fi + mkdir -p "${RUNNER_TEMP}/bench-live-reports" + cp "$cohort_source/bench-live-cohort.json" "${RUNNER_TEMP}/bench-live-cohort.json" + cp "$cohort_source/bench-live-cohort.attestation.json" "${RUNNER_TEMP}/bench-live-cohort.attestation.json" + for sample in 1 2 3 4 5; do + slot="$(printf '%02d' "$sample")" + source="${RUNNER_TEMP}/bench-live-reports-download/bench-live-sample-${GITHUB_RUN_ATTEMPT}-${sample}/slots/${slot}" + expected_inventory=$'attestation.bundle.json\nreceipt.json\nreport.json\nreservation.attestation.json\nreservation.receipt.json' + actual_inventory="$(find "$source" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort)" + test "$actual_inventory" = "$expected_inventory" || { + echo "::error::Release slot ${slot} has an unexpected artifact inventory." + exit 1 + } + mkdir -p "${RUNNER_TEMP}/bench-live-reports/slots/${slot}" + for name in report.json receipt.json reservation.receipt.json reservation.attestation.json attestation.bundle.json; do + test -f "$source/$name" && test ! -L "$source/$name" && test -s "$source/$name" || { + echo "::error::Missing ${name} for release slot ${slot}." + exit 1 + } + cp "$source/$name" "${RUNNER_TEMP}/bench-live-reports/slots/${slot}/${name}" + done + done + - name: Verify signed release benchmark evidence + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if [[ "${{ needs.bench-live-sample.result }}" != "success" ]]; then + echo "::error::At least one live benchmark sample failed before attestation verification." + exit 1 + fi + gh attestation verify "${RUNNER_TEMP}/bench-live-cohort.json" \ + --bundle "${RUNNER_TEMP}/bench-live-cohort.attestation.json" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/release.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref "${GITHUB_REF}" \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json > /dev/null + for slot in 01 02 03 04 05; do + report="${RUNNER_TEMP}/bench-live-reports/slots/${slot}/report.json" + receipt="${RUNNER_TEMP}/bench-live-reports/slots/${slot}/receipt.json" + reservation="${RUNNER_TEMP}/bench-live-reports/slots/${slot}/reservation.receipt.json" + reservation_bundle="${RUNNER_TEMP}/bench-live-reports/slots/${slot}/reservation.attestation.json" + bundle="${RUNNER_TEMP}/bench-live-reports/slots/${slot}/attestation.bundle.json" + test -s "$report" + test -s "$receipt" + test -s "$reservation" + test -s "$reservation_bundle" + test -s "$bundle" + gh attestation verify "$reservation" \ + --bundle "$reservation_bundle" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/release.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref "${GITHUB_REF}" \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json > /dev/null + jq -e -s ' + .[0].state == "running" and + .[1].state == "completed" and + .[0].manifestSha256 == .[1].manifestSha256 and + .[0].cohortId == .[1].cohortId and + .[0].purpose == .[1].purpose and + .[0].slot == .[1].slot and + .[0].nonce == .[1].nonce and + .[0].runId == .[1].runId and + .[0].startedAt == .[1].startedAt + ' "$reservation" "$receipt" >/dev/null || { + echo "::error::Release slot ${slot} terminal receipt is not bound to its reservation." + exit 1 + } + for subject in "$report" "$receipt"; do + gh attestation verify "$subject" \ + --bundle "$bundle" \ + --repo postil-dev/postil-cli \ + --signer-repo postil-dev/postil-cli \ + --signer-workflow postil-dev/postil-cli/.github/workflows/release.yml \ + --signer-digest "${GITHUB_SHA}" \ + --source-digest "${GITHUB_SHA}" \ + --source-ref "${GITHUB_REF}" \ + --cert-oidc-issuer https://token.actions.githubusercontent.com \ + --predicate-type https://slsa.dev/provenance/v1 \ + --deny-self-hosted-runners \ + --format json > /dev/null + done + done + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + - name: Verify independent release generations + working-directory: bench + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + bun run bench:verify-generations -- \ + --result "${RUNNER_TEMP}/bench-live-reports/slots/01/report.json" \ + --receipt "${RUNNER_TEMP}/bench-live-reports/slots/01/receipt.json" \ + --result "${RUNNER_TEMP}/bench-live-reports/slots/02/report.json" \ + --receipt "${RUNNER_TEMP}/bench-live-reports/slots/02/receipt.json" \ + --result "${RUNNER_TEMP}/bench-live-reports/slots/03/report.json" \ + --receipt "${RUNNER_TEMP}/bench-live-reports/slots/03/receipt.json" \ + --result "${RUNNER_TEMP}/bench-live-reports/slots/04/report.json" \ + --receipt "${RUNNER_TEMP}/bench-live-reports/slots/04/receipt.json" \ + --result "${RUNNER_TEMP}/bench-live-reports/slots/05/report.json" \ + --receipt "${RUNNER_TEMP}/bench-live-reports/slots/05/receipt.json" - name: Compare against the recorded baseline - if: always() working-directory: bench env: - SAMPLE_1_OUTCOME: ${{ steps.live-sample-1.outcome }} - SAMPLE_2_OUTCOME: ${{ steps.live-sample-2.outcome }} - SAMPLE_3_OUTCOME: ${{ steps.live-sample-3.outcome }} + SAMPLE_JOB_RESULT: ${{ needs.bench-live-sample.result }} run: | set -euo pipefail comparison=0 bun run bench:compare -- \ --binary "${{ github.workspace }}/target/release/postil" \ --screen-profile ../provisional-models.json \ - --expected-run-id "release-${{ github.ref_name }}-sample-1" \ - --expected-run-id "release-${{ github.ref_name }}-sample-2" \ - --expected-run-id "release-${{ github.ref_name }}-sample-3" \ - --result "${{ runner.temp }}/bench-live-report-1.json" \ - --result "${{ runner.temp }}/bench-live-report-2.json" \ - --result "${{ runner.temp }}/bench-live-report-3.json" || comparison=$? - if [[ "${SAMPLE_1_OUTCOME}" != "success" || \ - "${SAMPLE_2_OUTCOME}" != "success" || \ - "${SAMPLE_3_OUTCOME}" != "success" ]]; then + --cohort-manifest "${{ runner.temp }}/bench-live-cohort.json" \ + --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-01" \ + --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-02" \ + --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-03" \ + --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-04" \ + --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-05" \ + --result "${{ runner.temp }}/bench-live-reports/slots/01/report.json" \ + --result "${{ runner.temp }}/bench-live-reports/slots/02/report.json" \ + --result "${{ runner.temp }}/bench-live-reports/slots/03/report.json" \ + --result "${{ runner.temp }}/bench-live-reports/slots/04/report.json" \ + --result "${{ runner.temp }}/bench-live-reports/slots/05/report.json" \ + --receipt "${{ runner.temp }}/bench-live-reports/slots/01/receipt.json" \ + --receipt "${{ runner.temp }}/bench-live-reports/slots/02/receipt.json" \ + --receipt "${{ runner.temp }}/bench-live-reports/slots/03/receipt.json" \ + --receipt "${{ runner.temp }}/bench-live-reports/slots/04/receipt.json" \ + --receipt "${{ runner.temp }}/bench-live-reports/slots/05/receipt.json" || comparison=$? + if [[ "${SAMPLE_JOB_RESULT}" != "success" ]]; then echo "::error::At least one live benchmark sample failed before comparison." exit 1 fi @@ -248,7 +603,7 @@ jobs: if: matrix.target == 'x86_64-unknown-linux-gnu' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: benchmarked-x86_64-unknown-linux-gnu + name: benchmarked-x86_64-unknown-linux-gnu-${{ github.run_attempt }} path: target/${{ matrix.target }}/release - name: Restore benchmarked binary mode if: matrix.target == 'x86_64-unknown-linux-gnu' @@ -271,9 +626,11 @@ jobs: fi # Keyless Sigstore signature + certificate (provenance / authenticity). # Verify later with: + # RELEASE_TAG=v0.x.y + # RELEASE_COMMIT= # cosign verify-blob "$art" --signature "$art.sig" --certificate "$art.pem" \ - # --certificate-identity "https://github.com/postil-dev/postil-cli/.github/workflows/release.yml@refs/tags/$RELEASE" \ - # --certificate-github-workflow-sha "$CLI_REF" \ + # --certificate-identity "https://github.com/postil-dev/postil-cli/.github/workflows/release.yml@refs/tags/${RELEASE_TAG}" \ + # --certificate-github-workflow-sha "${RELEASE_COMMIT}" \ # --certificate-oidc-issuer https://token.actions.githubusercontent.com cosign sign-blob --yes \ --output-signature "$art.sig" \ diff --git a/Cargo.lock b/Cargo.lock index 104efc4..347d775 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -58,7 +58,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -92,6 +92,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -194,6 +205,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + [[package]] name = "clap" version = "4.6.1" @@ -225,7 +247,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -265,6 +287,16 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -290,6 +322,36 @@ dependencies = [ "libc", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.2.2" @@ -320,6 +382,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "deadpool" version = "0.12.3" @@ -379,7 +447,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -390,7 +458,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -399,6 +467,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -421,7 +495,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -532,7 +606,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -586,11 +660,23 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + [[package]] name = "globset" version = "0.4.18" @@ -641,6 +727,76 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "http" version = "1.4.2" @@ -868,11 +1024,27 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] [[package]] name = "is_terminal_polyfill" @@ -913,7 +1085,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -932,7 +1104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -995,6 +1167,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.32" @@ -1039,6 +1220,29 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -1075,6 +1279,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -1100,6 +1308,29 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1112,9 +1343,15 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "postil-cli" -version = "0.9.3" +version = "0.9.4" dependencies = [ "aho-corasick", "anyhow", @@ -1125,6 +1362,7 @@ dependencies = [ "fs2", "futures", "globset", + "hickory-resolver", "httpdate", "libc", "memmap2", @@ -1136,7 +1374,7 @@ dependencies = [ "sha1", "sha2", "similar", - "syn", + "syn 2.0.117", "tempfile", "time", "tokio", @@ -1199,6 +1437,17 @@ dependencies = [ "termtree", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1238,7 +1487,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -1279,6 +1528,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.4" @@ -1286,7 +1541,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1296,7 +1562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1308,6 +1574,21 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -1387,6 +1668,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -1426,7 +1713,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1471,7 +1758,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", @@ -1483,7 +1770,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1534,6 +1821,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1541,7 +1834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -1590,7 +1883,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1697,7 +1990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1729,6 +2022,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -1746,9 +2050,36 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.27.0" @@ -1759,7 +2090,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1785,7 +2116,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1867,7 +2198,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2044,6 +2375,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wait-timeout" version = "0.2.1" @@ -2129,7 +2471,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -2171,6 +2513,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -2193,7 +2541,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2208,6 +2556,35 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2437,7 +2814,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2458,7 +2835,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2478,7 +2855,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2518,7 +2895,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4f124a0..fc002e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.3" +version = "0.9.4" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" @@ -29,6 +29,7 @@ dirs = "6.0.0" fs2 = "0.4.3" futures = "0.3.32" globset = "0.4.18" +hickory-resolver = "0.26.1" httpdate = "1.0.3" libc = "0.2.186" memmap2 = "0.9.11" diff --git a/ROADMAP.md b/ROADMAP.md index 21cf532..9945da6 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,9 +39,8 @@ - Validate the Bitbucket and Azure DevOps incremental (`--since-sha`) diff paths against live instances. The full-PR-diff paths are exercised by tests; the incremental ones - depend on API conventions (Bitbucket's `diff/{spec}` two-dot order — which may also - apply merge-base semantics on Cloud; Azure's changed-file reconstruction) that we - have not yet confirmed end to end. + depend on unverified API conventions (Bitbucket's `diff/{spec}` two-dot order, which + may also apply merge-base semantics on Cloud; Azure's changed-file reconstruction). - Bitbucket inline-comment threading and Azure DevOps iteration-aware diffs for very large PRs. - Learning from dismissals: feed comment-resolution outcomes from the hosted platform @@ -52,8 +51,8 @@ ## Benchmarking status The hermetic PR-review benchmark harness lives in `bench/`: isolated run dirs, -mock forge and model endpoints, prompt-leakage guardrails, and 40 fixtures. The -set contains 33 seeded defects across languages and change classes plus 7 clean +mock forge and model endpoints, prompt-leakage guardrails, and 70 fixtures. The +set contains 57 seeded defects across languages and change classes plus 13 clean PRs where correct behavior is silence. Mock mode runs in CI against a release build and measures pipeline fidelity: @@ -61,11 +60,21 @@ grounding, gating, statusline correctness, and prompt-leakage controls. It does not measure detection ability because the mock model returns recorded findings generated from fixture specs. -Live-model mode is manual because it spends real model tokens. It runs the same -40 fixtures against selected OpenRouter-compatible models while keeping forge I/O -mocked, then reports detection rate, false positives, catalog-priced token-cost -estimates, and per-case detail. Diff-file live mode is available for single-model -local checks with no mock forge. +Live-model mode runs the same 70 fixtures against selected OpenRouter-compatible +models while keeping forge I/O mocked, then reports detection rate, false +positives, cost with provider or catalog-estimate provenance, latency, and +per-case detail. Local runs are explicit because they spend real model tokens. +Tagged releases run five sequential complete samples and compare their cohort +against a signed, predeclared ten-report calibration baseline. Durable slot +reservations, receipts, reports, cohort manifests, and binaries are +authenticated by GitHub OIDC attestations for release and calibration runs. +OpenRouter generation records independently bind each cohort to distinct +provider calls with matching model, provider, token, and cost totals. +Calibration runs once on the current `main` commit, with each reservation +attested before its GitHub-hosted job starts inference. Failed, interrupted, +missing, substituted, and copied samples invalidate the whole cohort rather +than becoming replaceable observations. Diff-file live mode is +available for single-model local checks with no mock forge. Comparative claims require peer runs on the identical fixture set; site comparisons stay qualitative and sourced until then. diff --git a/bench/README.md b/bench/README.md index 6725102..20bd231 100644 --- a/bench/README.md +++ b/bench/README.md @@ -376,7 +376,7 @@ exercise the scorer and must retain its exact identity and usage record. It refuses to run without `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`, or `LLM_API_KEY` and never logs or prints the key value. Live mode spends real tokens and depends on an external provider, so it is **not run on ordinary -pushes or pull requests**. The release pipeline runs three sequential +pushes or pull requests**. The release pipeline runs five sequential full-corpus live samples against `bench/baseline.json` before every tagged release (see "Release gate" below). Every live run writes its JSON report under `.runs/live//` (gitignored), beside raw per-attempt stdout and stderr. @@ -427,7 +427,7 @@ produced no valid v1 envelope at all (empty/garbled output, typically a dropped response). A valid envelope is always treated as a normal result and is never retried, including a gate-failing exit (exit 1 with a scored envelope) or one that merely reports findings unrelated to the authored target. A case that fails on both -attempts is recorded as an error and excluded from scoring, exactly as before. +attempts is recorded as an operational error and excluded from scoring. ### What live mode scores @@ -458,16 +458,40 @@ nondeterministic. Treat them as internal evidence, not a published benchmark. ## Release gate -The `Release` workflow runs three uniquely named, sequential full-corpus +The `Release` workflow runs five uniquely named, sequential full-corpus diff-file samples against the Luna profile in `provisional-models.json`. Every -sample uses the same preserved release binary. All three samples are attempted -even when one fails, and the workflow uploads every completed raw report before -comparison. A failed sample, missing report, or failed comparison blocks the -release because `build` depends on the `bench-live` job. - -The comparator accepts exactly one or three distinct `--result` paths. Three -paths must also contain byte-distinct reports with distinct immutable run IDs -and timestamps. Byte identity is established by SHA-256 over each raw file. +sample uses the same preserved release binary. All five samples are attempted +even when one fails. The prepare job writes one five-slot cohort manifest bound +to the source commit, tag ref, workflow run, and attempt. GitHub OIDC attests the +exact manifest with public Sigstore provenance. Each sample reserves one +canonical slot directory and attests the running receipt before inference, +then attests its completed report and receipt together. Every accepted provider +response contributes its OpenRouter generation ID to the report. The fan-in job +verifies globally distinct generation IDs against OpenRouter's authenticated +generation API, including the exact model, provider, token totals, and cost, +then verifies every subject against the exact repository, release workflow, +source commit, tag ref, OIDC issuer, and GitHub-hosted runner before parsing it. +Only the unique first workflow run for +the version tag is authoritative. Tag-scoped concurrency, an existing-release +check, and duplicate-run rejection prevent a second publisher path. A failed +sample retains its terminal receipt but has no successful attestation. A +failed sample, missing or unverifiable subject, incomplete slot, or failed +comparison blocks the release because `build` depends on the `bench-live` job. + +The comparator accepts exactly one, three, or five distinct `--result` paths for +comparison. One- and three-report comparisons support smaller local checks; the +release workflow always supplies five. Multi-report paths must contain +byte-distinct reports with distinct immutable run IDs and timestamps. Byte +identity is established by SHA-256 over each raw file and authenticated by the +release attestations. +Five-report comparisons additionally require the original manifest and one +completed receipt for every slot. The comparator verifies the manifest against +the supplied binary, evaluator, corpus, profile, provider contract, workflow +run, and workflow attempt. It verifies every receipt's slot, nonce, run ID, +report digest, and timestamp interval. Semantically identical outcomes are +valid when their raw subjects have independent authenticated provenance. +Running, failed, missing, substituted, duplicate, and extra slots invalidate +the whole cohort. Every report must be exhaustive full-corpus evidence with an empty `selectedCaseIds`, an enforced provider contract, no operational errors, and all cases scored. Summary and per-result cost accounting must be complete. @@ -480,28 +504,53 @@ binary and screening-profile paths, then recomputes the binary, fixture corpus, evaluator source, screening profile, provider contract, and exact case cohort instead of trusting hashes asserted by the reports. -A three-report comparison also requires identical binary, corpus, evaluator, +A five-report comparison also requires identical binary, corpus, evaluator, model, provider, API, scorer, route, profile, contract, timeout, fixture -identity, and case-count fields across the cohort. Structural, operational, -digest, and cohort failures block before metric comparison. The one-report -mode applies the same fail-closed report validation. - -`compare-baseline.ts` compares five aggregate metrics against the matching -model entry in `bench/baseline.json`: median authored-target detection rate, -median false/unrelated finding count, median gate-verdict correctness, maximum -per-run mean provider cost per case, and median per-run nearest-rank p95 review -latency. Each metric has its own tolerance in the exported `*_MAX_*` constants. -Detection rate and p95 latency block when they cross their tolerances. Maximum -mean cost always blocks and is compared as an exact decimal ratio; a provider -profile mismatch invalidates the comparison instead of making cost -informational. False/unrelated findings and gate-verdict correctness remain -informational; their medians and complete observed ranges appear in the table. -The CLI's per-operation cost cap remains the deterministic spending boundary. - -The baseline records fixture-corpus and evaluator-source SHA-256 digests, the -expected complete case count, exact provider profile, and the maximum sampled -run cost as a canonical decimal with its case count. A mismatch blocks -comparison across unrelated or incomplete evidence. +identity, and case-count fields across the release cohort. Structural, +operational, digest, and cohort failures block before metric comparison. Every +report count applies the same fail-closed report validation. + +`compare-baseline.ts` compares five aggregate metrics against the matching model +entry in `bench/baseline.json`: mean authored-target detection, median false or +unrelated finding count, median gate-verdict correctness, maximum per-run mean +provider cost per case, and median per-run nearest-rank p95 review latency. +Detection uses exact count arithmetic over the 57 defect fixtures. A five-report +release candidate passes the detection non-inferiority check when its cohort +mean is no more than two defect detections below the recorded calibration mean. +The two-defect margin is applied to counts, not a rounded percentage. Detection, +p95 latency, and cost are blocking checks; false findings and gate-verdict +correctness remain informational with their medians and complete observed +ranges. The CLI's per-operation cost cap remains the deterministic spending +boundary. + +Baseline recording uses a predeclared calibration cohort of exactly ten +independent complete reports from one frozen binary, corpus, evaluator, provider +profile, and case cohort. A calibration report is not replaced because its +outcome is inconvenient: the ten-report cohort is fixed before execution, and +missing, duplicate, failed, interrupted, or incomplete evidence fails closed. +The `Benchmark calibration` workflow runs only once for the current `main` +commit. Before model execution it creates an immutable, server-protected +`postil-calibration-` registry tag; a failed source cannot be +rerun. It builds and attests one release binary and the ten-slot manifest. +Each slot runs in a separate GitHub-hosted job. The job attests its running +receipt before inference starts, executes the full corpus, and attests the +terminal report and receipt. The fan-in job verifies every offline Sigstore +bundle against the exact repository, workflow, source commit, branch, OIDC +issuer, and GitHub-hosted runner. It also verifies the running-to-completed +receipt transition and independently audits every globally distinct provider +generation before recording the baseline as a workflow artifact. The +baseline records the manifest and source digests, workflow run identity, +each slot and nonce, report and receipt SHA-256 values, normalized outcome +digests, per-run metric distribution, fixture-corpus and evaluator-source +digests, complete case counts, exact provider profile, and the maximum sampled +run cost as a canonical decimal with its case count. Checksums bind content; +GitHub attestations authenticate build and execution provenance. A release +candidate requires the populated baseline and its attestation bundle committed +together. The release verifies that bundle against the calibration workflow, +the recorded source commit, and the immutable calibration registry tag before +using any threshold. The candidate must match the recorded corpus, evaluator, +provider profile, and case cohort. Its five reports must share one candidate +binary. A mismatch blocks comparison across unrelated or incomplete evidence. ```sh # Compare one complete report for a local check. @@ -511,32 +560,88 @@ bun run bench:compare -- \ --expected-run-id \ --result -# Run the release comparison over three complete reports. +# Inside the release workflow, create and attest a run-bound manifest before +# any candidate sample starts, then execute each canonical slot once. +bun run bench:cohort-create -- \ + --purpose release \ + --binary \ + --screen-profile ../provisional-models.json \ + --run-prefix \ + --out +bun run bench:cohort-run -- \ + --mode reserve \ + --manifest \ + --slot <1-through-5> \ + --binary \ + --screen-profile ../provisional-models.json +bun run bench:cohort-run -- \ + --mode execute \ + --manifest \ + --slot <1-through-5> \ + --binary \ + --screen-profile ../provisional-models.json + +# Run the release comparison over five complete candidate reports and receipts. bun run bench:compare -- \ --binary \ --screen-profile ../provisional-models.json \ + --cohort-manifest \ --expected-run-id \ --expected-run-id \ --expected-run-id \ + --expected-run-id \ + --expected-run-id \ --result \ --result \ - --result - -# Re-baseline explicitly from three independent complete reports with the same -# binary, profile, and corpus. + --result \ + --result \ + --result \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt + +# The Benchmark calibration workflow invokes the record operation after it +# verifies the attested binary, manifest, reservations, reports, and receipts. bun run bench:compare -- \ --binary \ --screen-profile ../provisional-models.json \ + --cohort-manifest \ --expected-run-id \ --expected-run-id \ --expected-run-id \ + --expected-run-id \ + --expected-run-id \ + --expected-run-id \ + --expected-run-id \ + --expected-run-id \ + --expected-run-id \ + --expected-run-id \ --result \ --result \ --result \ + --result \ + --result \ + --result \ + --result \ + --result \ + --result \ + --result \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ + --receipt \ --record ``` -`--record` accepts exactly three reports and is the only operation that writes +`--record` accepts exactly ten reports and is the only operation that writes `bench/baseline.json`. The release workflow never records a baseline. The -comparison table shows baseline, aggregate observation, verdict, and sample -range directly in the job log. +comparison table shows the calibration baseline, candidate cohort observation, +verdict, and complete sample range directly in the job log. diff --git a/bench/baseline.json b/bench/baseline.json index 48b5a7b..19d8558 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -1,8 +1,8 @@ { - "schemaVersion": 1, + "schemaVersion": 2, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "1d6f9fecda01e6e8066f8fefd4a1b2955014575ce07542819b6f506d48dae595" + "evaluatorSha256": "ca71c7172e61cb2fd5c11037117ae8003b5e3b18ab9ff2cea55943438f1ba75e" }, "profiles": { "z-ai/glm-5.2": { @@ -25,25 +25,8 @@ } }, "openai/gpt-5.6-luna": { - "populated": true, - "generatedAt": "2026-08-25T20:48:41.276Z", - "reviewMode": "exhaustive", - "sourceRunAt": "2026-08-25T20:37:29.452Z", - "providerContractEnforced": true, - "screeningProfileSha256": "aea05c3f5622cebec480d2a8daf5bb53055bc0160206e6f22e889391ea71fa49", - "upstreamProviderIdentity": "Azure", - "totalCases": 70, - "scoredCases": 70, - "detectionRate": 0.9473684210526315, - "falsePositives": 0, - "gateVerdictCorrectness": 0.8285714285714286, - "meanCostUsdPerCase": 0.0007761659714285714, - "maximumRunCostUsdDecimal": "0.054331618", - "costCaseCount": 70, - "latencyMs": { - "p50": 5240, - "p95": 9363 - } + "populated": false, + "instructions": "Record a signed, predeclared ten-slot calibration cohort before release comparison." } } } diff --git a/bench/evaluator-contract-sources.json b/bench/evaluator-contract-sources.json index e257715..89f2339 100644 --- a/bench/evaluator-contract-sources.json +++ b/bench/evaluator-contract-sources.json @@ -1,4 +1,6 @@ [ + ".github/workflows/benchmark-calibration.yml", + ".github/workflows/release.yml", "bench/admission-manifest-candidate-vector.json", "bench/evaluator-contract-sources.json", "bench/package.json", @@ -7,7 +9,12 @@ "bench/fixtures/cases.ts", "bench/src/api-key.ts", "bench/src/attribution.ts", + "bench/src/cohort.ts", + "bench/src/cohort-run.ts", + "bench/src/compare-baseline.ts", + "bench/src/generation-evidence.ts", "bench/src/harness.ts", + "bench/src/live.ts", "bench/src/livemodels-score.ts", "bench/src/livemodels.ts", "bench/src/request-window.ts", diff --git a/bench/package.json b/bench/package.json index 762ea39..2850e85 100644 --- a/bench/package.json +++ b/bench/package.json @@ -9,6 +9,9 @@ "bench:live": "bun run src/run.ts --live", "bench:live-models": "POSTIL_BENCH_MODE=live bun run src/run.ts", "bench:compare": "bun run src/compare-baseline.ts", + "bench:cohort-create": "bun run src/cohort.ts", + "bench:cohort-run": "bun run src/cohort-run.ts", + "bench:verify-generations": "bun run src/generation-evidence.ts", "verify-admission": "bun run src/verify-admission.ts", "scorer-eval": "bun run src/scorer-eval.ts" }, diff --git a/bench/review-contract-sources.json b/bench/review-contract-sources.json index 8ce5eb1..0c420d1 100644 --- a/bench/review-contract-sources.json +++ b/bench/review-contract-sources.json @@ -3,6 +3,7 @@ "Cargo.toml", "build.rs", "src/adjudication.rs", + "src/alerts.rs", "src/api_key.rs", "src/attribution.rs", "src/brevity.rs", diff --git a/bench/src/cohort-run.ts b/bench/src/cohort-run.ts new file mode 100644 index 0000000..c04f0e7 --- /dev/null +++ b/bench/src/cohort-run.ts @@ -0,0 +1,286 @@ +#!/usr/bin/env bun +// Reserves or executes exactly one predeclared benchmark slot. + +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { basename, dirname, resolve } from "node:path"; +import { + assertManifestBoundToInputs, + cohortReceiptSchema, + cohortSlotPaths, + readCohortManifest, + sha256, + type CohortManifest, + type CohortReceipt, + type CohortSlot, +} from "./cohort"; + +interface SlotOptions { + manifestPath: string; + slot: number; + binaryPath: string; + screeningProfilePath: string; + environment?: NodeJS.ProcessEnv; +} + +interface SlotContext { + manifest: CohortManifest; + manifestSha256: string; + cohortSlot: CohortSlot; + paths: ReturnType; +} + +async function slotContext(options: SlotOptions): Promise { + const { manifest, rawSha256: manifestSha256 } = await readCohortManifest(options.manifestPath); + const cohortSlot = manifest.slots.find((candidate) => candidate.slot === options.slot); + if (cohortSlot === undefined) throw new Error(`cohort manifest has no slot ${options.slot}`); + await assertManifestBoundToInputs( + manifest, + options.binaryPath, + options.screeningProfilePath, + options.environment, + ); + return { + manifest, + manifestSha256, + cohortSlot, + paths: cohortSlotPaths(options.manifestPath, options.slot), + }; +} + +function runningReceipt(context: SlotContext): CohortReceipt { + return cohortReceiptSchema.parse({ + schemaVersion: 2, + state: "running", + manifestSha256: context.manifestSha256, + cohortId: context.manifest.cohortId, + purpose: context.manifest.purpose, + slot: context.cohortSlot.slot, + nonce: context.cohortSlot.nonce, + runId: context.cohortSlot.runId, + startedAt: new Date().toISOString(), + }); +} + +function assertReservedReceipt(context: SlotContext, receipt: CohortReceipt): asserts receipt is Extract< + CohortReceipt, + { state: "running" } +> { + if (receipt.state !== "running") { + throw new Error(`cohort slot ${context.cohortSlot.slot} reservation is already ${receipt.state}`); + } + for (const [field, actual, expected] of [ + ["manifestSha256", receipt.manifestSha256, context.manifestSha256], + ["cohortId", receipt.cohortId, context.manifest.cohortId], + ["purpose", receipt.purpose, context.manifest.purpose], + ["slot", receipt.slot, context.cohortSlot.slot], + ["nonce", receipt.nonce, context.cohortSlot.nonce], + ["runId", receipt.runId, context.cohortSlot.runId], + ] as const) { + if (actual !== expected) { + throw new Error(`cohort slot ${context.cohortSlot.slot} reserved ${field} does not match its manifest`); + } + } +} + +async function replaceReceipt(path: string, receipt: CohortReceipt): Promise { + const parsed = cohortReceiptSchema.parse(receipt); + const temporary = resolve(dirname(path), `.${basename(path)}.${process.pid}.${randomUUID()}.tmp`); + await writeFile(temporary, `${JSON.stringify(parsed, null, 2)}\n`, { mode: 0o600 }); + await rename(temporary, path); +} + +async function reportDigests(path: string, expectedRunId: string): Promise<{ + rawSha256: string; + ranAt: string; +}> { + const raw = await readFile(path); + let parsed: unknown; + try { + parsed = JSON.parse(raw.toString("utf8")); + } catch (error) { + throw new Error(`could not parse benchmark report: ${error instanceof Error ? error.message : String(error)}`); + } + const runId = (parsed as { summary?: { runId?: unknown } })?.summary?.runId; + if (runId !== expectedRunId) throw new Error("benchmark report runId does not match its cohort slot"); + const ranAt = (parsed as { summary?: { ranAt?: unknown } })?.summary?.ranAt; + if (typeof ranAt !== "string" || !Number.isFinite(Date.parse(ranAt))) { + throw new Error("benchmark report contains an invalid ranAt timestamp"); + } + return { rawSha256: sha256(raw), ranAt }; +} + +export async function reserveCohortSlot(options: SlotOptions): Promise { + const context = await slotContext(options); + await mkdir(resolve(context.paths.directory, ".."), { recursive: true, mode: 0o700 }); + await mkdir(context.paths.directory, { mode: 0o700 }); + const receipt = runningReceipt(context); + await writeFile(context.paths.receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); + return receipt; +} + +export async function executeReservedCohortSlot(options: SlotOptions & { + executeBenchmark?: (options: { reportPath: string; runId: string }) => Promise; +}): Promise { + const context = await slotContext(options); + const receiptRaw = await readFile(context.paths.receiptPath).catch((error) => { + throw new Error( + `cohort slot ${context.cohortSlot.slot} has no authenticated reservation: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + const reservation = cohortReceiptSchema.parse(JSON.parse(receiptRaw.toString("utf8"))); + assertReservedReceipt(context, reservation); + + const finishReceipt = async (receipt: CohortReceipt): Promise => { + await replaceReceipt(context.paths.receiptPath, receipt); + }; + + let exitCode: number | null = null; + try { + if (options.executeBenchmark !== undefined) { + exitCode = await options.executeBenchmark({ + reportPath: context.paths.reportPath, + runId: context.cohortSlot.runId, + }); + } else { + const child = Bun.spawn([ + process.execPath, + "run", + "src/run.ts", + "--live", + "--screen-profile", + options.screeningProfilePath, + "--run-id", + context.cohortSlot.runId, + "--json-out", + context.paths.reportPath, + ], { + cwd: resolve(import.meta.dir, ".."), + env: { ...process.env, POSTIL_BIN: options.binaryPath }, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }); + exitCode = await child.exited; + } + } catch (error) { + await finishReceipt({ + ...reservation, + state: "failed", + finishedAt: new Date().toISOString(), + exitCode: null, + failure: "spawn-failed", + reportRawSha256: null, + }); + throw error; + } + + let digests: Awaited> | undefined; + try { + digests = await reportDigests(context.paths.reportPath, context.cohortSlot.runId); + } catch (error) { + await finishReceipt({ + ...reservation, + state: "failed", + finishedAt: new Date().toISOString(), + exitCode, + failure: exitCode === 0 ? "report-invalid" : "benchmark-exit", + reportRawSha256: null, + }); + if (exitCode === 0) throw error; + return exitCode; + } + + if (exitCode !== 0) { + await finishReceipt({ + ...reservation, + state: "failed", + finishedAt: new Date().toISOString(), + exitCode, + failure: "benchmark-exit", + reportRawSha256: digests.rawSha256, + }); + return exitCode; + } + + const finishedAt = new Date().toISOString(); + if ( + Date.parse(digests.ranAt) < Date.parse(reservation.startedAt) || + Date.parse(digests.ranAt) > Date.parse(finishedAt) + ) { + await finishReceipt({ + ...reservation, + state: "failed", + finishedAt, + exitCode: 0, + failure: "report-invalid", + reportRawSha256: digests.rawSha256, + }); + throw new Error("benchmark report ranAt is outside its receipt interval"); + } + await finishReceipt({ + ...reservation, + state: "completed", + finishedAt, + exitCode: 0, + reportRawSha256: digests.rawSha256, + }); + return 0; +} + +export async function runCohortSlot(options: SlotOptions & { + executeBenchmark?: (options: { reportPath: string; runId: string }) => Promise; +}): Promise { + await reserveCohortSlot(options); + return executeReservedCohortSlot(options); +} + +function requiredValue(args: readonly string[], index: number, flag: string): string { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${flag} requires a value`); + return value; +} + +async function main(): Promise { + const args = process.argv.slice(2); + const values = new Map(); + const allowed = new Set(["--mode", "--manifest", "--slot", "--binary", "--screen-profile"]); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]!; + if (!allowed.has(flag)) throw new Error(`unknown cohort-run argument ${flag}`); + if (values.has(flag)) throw new Error(`${flag} may be specified only once`); + values.set(flag, requiredValue(args, index, flag)); + } + for (const flag of allowed) { + if (!values.has(flag)) throw new Error(`cohort-run requires ${flag}`); + } + const mode = values.get("--mode"); + if (mode !== "reserve" && mode !== "execute") { + throw new Error("--mode must be reserve or execute"); + } + const slot = Number(values.get("--slot")); + if (!Number.isSafeInteger(slot) || slot < 1) throw new Error("cohort slot must be a positive integer"); + const options = { + manifestPath: resolve(values.get("--manifest")!), + slot, + binaryPath: resolve(values.get("--binary")!), + screeningProfilePath: resolve(values.get("--screen-profile")!), + }; + if (mode === "reserve") { + await reserveCohortSlot(options); + } else { + process.exitCode = await executeReservedCohortSlot(options); + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts new file mode 100644 index 0000000..f8ea2d5 --- /dev/null +++ b/bench/src/cohort.test.ts @@ -0,0 +1,254 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { + assertManifestBoundToInputs, + cohortSlotPaths, + cohortManifestSchema, + createCohortManifest, + readCohortReceipt, + reportSemanticSha256, + type CohortManifest, +} from "./cohort"; +import { executeReservedCohortSlot, reserveCohortSlot } from "./cohort-run"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true }))); +}); + +async function temporaryDirectory(): Promise { + const path = await mkdtemp(join(tmpdir(), "postil-cohort-test-")); + temporaryDirectories.push(path); + return path; +} + +const screeningProfilePath = resolve(import.meta.dir, "..", "..", "provisional-models.json"); + +function calibrationExecution(): CohortManifest["execution"] { + return { + kind: "github-sigstore-v1", + repository: "postil-dev/postil-cli", + signerWorkflow: ".github/workflows/benchmark-calibration.yml", + sourceSha: "b".repeat(40), + sourceRef: "refs/heads/main", + runId: "456", + runAttempt: "1", + }; +} + +function githubEnvironment(execution: CohortManifest["execution"]): NodeJS.ProcessEnv { + return { + GITHUB_REPOSITORY: execution.repository, + GITHUB_SHA: execution.sourceSha, + GITHUB_REF: execution.sourceRef, + GITHUB_RUN_ID: execution.runId, + GITHUB_RUN_ATTEMPT: execution.runAttempt, + }; +} + +describe("cohort manifests", () => { + test("predeclares exactly ten immutable calibration slots", async () => { + let sequence = 0; + const execution = calibrationExecution(); + const manifest = await createCohortManifest({ + purpose: "calibration", + binaryPath: process.execPath, + screeningProfilePath, + runPrefix: "calibration-e", + execution, + now: new Date("2026-08-26T00:00:00.000Z"), + uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, + }); + expect(manifest.reportCount).toBe(10); + expect(manifest.slots.map((slot) => slot.slot)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(manifest.slots.map((slot) => slot.runId)).toEqual([ + "calibration-e-01", "calibration-e-02", "calibration-e-03", "calibration-e-04", + "calibration-e-05", "calibration-e-06", "calibration-e-07", "calibration-e-08", + "calibration-e-09", "calibration-e-10", + ]); + await expect( + assertManifestBoundToInputs( + manifest, + process.execPath, + screeningProfilePath, + githubEnvironment(execution), + ), + ).resolves.toBeUndefined(); + + const tampered = structuredClone(manifest); + tampered.evaluatorSha256 = "f".repeat(64); + await expect( + assertManifestBoundToInputs( + tampered, + process.execPath, + screeningProfilePath, + githubEnvironment(execution), + ), + ).rejects.toThrow("evaluatorSha256 is not bound"); + }); + + test("rejects wrong counts, unordered slots, and unbound release execution", async () => { + let sequence = 100; + const execution = calibrationExecution(); + const calibration = await createCohortManifest({ + purpose: "calibration", + binaryPath: process.execPath, + screeningProfilePath, + runPrefix: "calibration", + execution, + uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, + }); + expect(() => cohortManifestSchema.parse({ + ...calibration, + purpose: "release", + })).toThrow("release cohorts require exactly 5 reports"); + expect(() => cohortManifestSchema.parse({ + ...calibration, + slots: [calibration.slots[1], calibration.slots[0], ...calibration.slots.slice(2)], + })).toThrow("ordered and contiguous"); + }); + + test("binds GitHub release execution to the first run attempt", async () => { + let sequence = 200; + const manifest = await createCohortManifest({ + purpose: "release", + binaryPath: process.execPath, + screeningProfilePath, + runPrefix: "release", + execution: { + kind: "github-sigstore-v1", + repository: "postil-dev/postil-cli", + signerWorkflow: ".github/workflows/release.yml", + sourceSha: "a".repeat(40), + sourceRef: "refs/tags/v0.9.4", + runId: "123", + runAttempt: "1", + }, + uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, + }); + await expect(assertManifestBoundToInputs( + manifest, + process.execPath, + screeningProfilePath, + { + GITHUB_REPOSITORY: "postil-dev/postil-cli", + GITHUB_SHA: "a".repeat(40), + GITHUB_REF: "refs/tags/v0.9.4", + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "1", + }, + )).resolves.toBeUndefined(); + await expect(assertManifestBoundToInputs( + manifest, + process.execPath, + screeningProfilePath, + { + GITHUB_REPOSITORY: "postil-dev/postil-cli", + GITHUB_SHA: "a".repeat(40), + GITHUB_REF: "refs/tags/v0.9.4", + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "2", + }, + )).rejects.toThrow("not bound to this GitHub Actions source, run, and attempt"); + }); +}); + +test("semantic digest excludes execution noise", () => { + const original = { + summary: { runId: "one", ranAt: "2026-08-26T00:00:00.000Z", durationMs: 10 }, + results: [{ id: "case", detected: true, durationMs: 9 }], + }; + const renamed = structuredClone(original); + renamed.summary.runId = "two"; + renamed.summary.ranAt = "2026-08-26T00:01:00.000Z"; + expect(reportSemanticSha256(renamed)).toBe(reportSemanticSha256(original)); + renamed.results[0]!.durationMs += 1; + expect(reportSemanticSha256(renamed)).toBe(reportSemanticSha256(original)); + renamed.results[0]!.detected = false; + expect(reportSemanticSha256(renamed)).not.toBe(reportSemanticSha256(original)); +}); + +test("the canonical slot directory permanently blocks a path-based rerun", async () => { + const directory = await temporaryDirectory(); + let sequence = 300; + const execution = calibrationExecution(); + const manifest = await createCohortManifest({ + purpose: "calibration", + binaryPath: process.execPath, + screeningProfilePath, + runPrefix: "calibration", + execution, + uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, + }); + const manifestPath = join(directory, "manifest.json"); + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + await mkdir(join(directory, "slots", "01"), { recursive: true }); + await expect(reserveCohortSlot({ + manifestPath, + slot: 1, + binaryPath: process.execPath, + screeningProfilePath, + environment: githubEnvironment(execution), + })).rejects.toThrow("EEXIST"); +}); + +test("an authenticated reservation is required before slot execution", async () => { + const directory = await temporaryDirectory(); + let sequence = 400; + const execution = calibrationExecution(); + const manifest = await createCohortManifest({ + purpose: "calibration", + binaryPath: process.execPath, + screeningProfilePath, + runPrefix: "calibration", + execution, + uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, + }); + const manifestPath = join(directory, "manifest.json"); + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + const environment = githubEnvironment(execution); + + await expect(executeReservedCohortSlot({ + manifestPath, + slot: 1, + binaryPath: process.execPath, + screeningProfilePath, + environment, + executeBenchmark: async () => 0, + })).rejects.toThrow("no authenticated reservation"); + + const reservation = await reserveCohortSlot({ + manifestPath, + slot: 1, + binaryPath: process.execPath, + screeningProfilePath, + environment, + }); + expect(reservation.state).toBe("running"); + await expect(executeReservedCohortSlot({ + manifestPath, + slot: 1, + binaryPath: process.execPath, + screeningProfilePath, + environment, + executeBenchmark: async ({ reportPath, runId }) => { + await writeFile(reportPath, JSON.stringify({ + summary: { runId, ranAt: new Date().toISOString() }, + })); + return 0; + }, + })).resolves.toBe(0); + const receipt = await readCohortReceipt(cohortSlotPaths(manifestPath, 1).receiptPath); + expect(receipt.receipt.state).toBe("completed"); + await expect(executeReservedCohortSlot({ + manifestPath, + slot: 1, + binaryPath: process.execPath, + screeningProfilePath, + environment, + executeBenchmark: async () => 0, + })).rejects.toThrow("already completed"); +}, 20_000); diff --git a/bench/src/cohort.ts b/bench/src/cohort.ts new file mode 100644 index 0000000..b00c2cd --- /dev/null +++ b/bench/src/cohort.ts @@ -0,0 +1,375 @@ +#!/usr/bin/env bun +// Predeclared execution contracts for release benchmark cohorts. + +import { createHash, randomUUID } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { z } from "zod"; +import { cases } from "../fixtures/cases"; +import { benchmarkCase } from "./harness"; +import { evaluatorSourceSha256, screeningProfileMetadata } from "./live"; + +const sha256Schema = z.string().regex(/^[0-9a-f]{64}$/u); +const gitShaSchema = z.string().regex(/^[0-9a-f]{40,64}$/u); +const nonemptyStringSchema = z.string().trim().min(1); +const REPOSITORY = "postil-dev/postil-cli"; +const RELEASE_WORKFLOW = ".github/workflows/release.yml"; +const CALIBRATION_WORKFLOW = ".github/workflows/benchmark-calibration.yml"; + +export const cohortSlotSchema = z.object({ + slot: z.number().int().positive(), + runId: nonemptyStringSchema, + nonce: z.string().uuid(), +}).strict(); + +const executionBindingSchema = z.object({ + kind: z.literal("github-sigstore-v1"), + repository: z.literal(REPOSITORY), + signerWorkflow: z.enum([RELEASE_WORKFLOW, CALIBRATION_WORKFLOW]), + sourceSha: gitShaSchema, + sourceRef: nonemptyStringSchema, + runId: nonemptyStringSchema, + runAttempt: z.literal("1"), +}).strict(); + +export const cohortManifestSchema = z.object({ + schemaVersion: z.literal(2), + purpose: z.enum(["calibration", "release"]), + cohortId: z.string().uuid(), + createdAt: z.string().datetime({ offset: true }), + reportCount: z.union([z.literal(5), z.literal(10)]), + binarySha256: sha256Schema, + evaluatorSha256: sha256Schema, + fixtureCorpusSha256: sha256Schema, + screeningProfileSha256: sha256Schema, + providerContractSha256: sha256Schema, + execution: executionBindingSchema, + slots: z.array(cohortSlotSchema), +}).strict().superRefine((manifest, context) => { + const expectedCount = manifest.purpose === "calibration" ? 10 : 5; + if (manifest.reportCount !== expectedCount) { + context.addIssue({ + code: "custom", + path: ["reportCount"], + message: `${manifest.purpose} cohorts require exactly ${expectedCount} reports`, + }); + } + if (manifest.slots.length !== manifest.reportCount) { + context.addIssue({ + code: "custom", + path: ["slots"], + message: "slot count must equal reportCount", + }); + } + const expectedSlots = Array.from({ length: manifest.reportCount }, (_, index) => index + 1); + if (manifest.slots.some((slot, index) => slot.slot !== expectedSlots[index])) { + context.addIssue({ + code: "custom", + path: ["slots"], + message: "slots must be ordered and contiguous starting at 1", + }); + } + if (new Set(manifest.slots.map((slot) => slot.runId)).size !== manifest.slots.length) { + context.addIssue({ code: "custom", path: ["slots"], message: "slot run IDs must be unique" }); + } + if (new Set(manifest.slots.map((slot) => slot.nonce)).size !== manifest.slots.length) { + context.addIssue({ code: "custom", path: ["slots"], message: "slot nonces must be unique" }); + } + if ( + manifest.purpose === "release" && + ( + manifest.execution.signerWorkflow !== RELEASE_WORKFLOW || + !/^refs\/tags\/v[^\s]+$/u.test(manifest.execution.sourceRef) + ) + ) { + context.addIssue({ + code: "custom", + path: ["execution"], + message: "release cohorts must be bound to the release workflow and a version tag", + }); + } + if ( + manifest.purpose === "calibration" && + ( + manifest.execution.signerWorkflow !== CALIBRATION_WORKFLOW || + manifest.execution.sourceRef !== "refs/heads/main" + ) + ) { + context.addIssue({ + code: "custom", + path: ["execution"], + message: "calibration cohorts must be bound to the main-branch calibration workflow", + }); + } +}); + +export type CohortManifest = z.infer; +export type CohortSlot = z.infer; + +const receiptBaseSchema = z.object({ + schemaVersion: z.literal(2), + manifestSha256: sha256Schema, + cohortId: z.string().uuid(), + purpose: z.enum(["calibration", "release"]), + slot: z.number().int().positive(), + nonce: z.string().uuid(), + runId: nonemptyStringSchema, + startedAt: z.string().datetime({ offset: true }), +}).strict(); + +export const cohortReceiptSchema = z.discriminatedUnion("state", [ + receiptBaseSchema.extend({ + state: z.literal("running"), + }).strict(), + receiptBaseSchema.extend({ + state: z.literal("completed"), + finishedAt: z.string().datetime({ offset: true }), + exitCode: z.literal(0), + reportRawSha256: sha256Schema, + }).strict(), + receiptBaseSchema.extend({ + state: z.literal("failed"), + finishedAt: z.string().datetime({ offset: true }), + exitCode: z.number().int().nullable(), + failure: z.enum(["spawn-failed", "benchmark-exit", "report-unavailable", "report-invalid"]), + reportRawSha256: sha256Schema.nullable(), + }).strict(), +]); + +export type CohortReceipt = z.infer; +export type CompletedCohortReceipt = Extract; + +export function sha256(bytes: Uint8Array | string): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function fixtureCorpusSha256(): string { + return sha256(JSON.stringify(cases.map((input) => benchmarkCase.parse(input)))); +} + +export function reportSemanticSha256(value: unknown): string { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("benchmark report must be a JSON object"); + } + const report = structuredClone(value) as Record; + if (typeof report.summary !== "object" || report.summary === null || Array.isArray(report.summary)) { + throw new Error("benchmark report summary must be a JSON object"); + } + const summary = report.summary as Record; + for (const field of [ + "runId", + "ranAt", + "durationMs", + "observedProviderCostUsdDecimal", + ]) delete summary[field]; + if (Array.isArray(report.results)) { + report.results = report.results.map((result) => { + if (typeof result !== "object" || result === null || Array.isArray(result)) return result; + const normalized = { ...result } as Record; + for (const field of [ + "durationMs", + "observedProviderCostUsdDecimal", + "promptTokens", + "completionTokens", + ]) delete normalized[field]; + return normalized; + }); + } + return sha256(JSON.stringify(report)); +} + +export function cohortSlotPaths(manifestPath: string, slot: number): { + directory: string; + reportPath: string; + receiptPath: string; +} { + if (!Number.isSafeInteger(slot) || slot < 1 || slot > 99) { + throw new Error("cohort slot must be an integer from 1 to 99"); + } + const directory = resolve(dirname(manifestPath), "slots", String(slot).padStart(2, "0")); + return { + directory, + reportPath: join(directory, "report.json"), + receiptPath: join(directory, "receipt.json"), + }; +} + +export async function readCohortManifest(path: string): Promise<{ + manifest: CohortManifest; + raw: Uint8Array; + rawSha256: string; +}> { + const raw = await readFile(path); + let parsed: unknown; + try { + parsed = JSON.parse(raw.toString("utf8")); + } catch (error) { + throw new Error(`could not parse cohort manifest: ${error instanceof Error ? error.message : String(error)}`); + } + return { + manifest: cohortManifestSchema.parse(parsed), + raw, + rawSha256: sha256(raw), + }; +} + +export async function readCohortReceipt(path: string): Promise<{ + receipt: CohortReceipt; + raw: Uint8Array; + rawSha256: string; +}> { + const raw = await readFile(path); + let parsed: unknown; + try { + parsed = JSON.parse(raw.toString("utf8")); + } catch (error) { + throw new Error(`could not parse cohort receipt: ${error instanceof Error ? error.message : String(error)}`); + } + return { + receipt: cohortReceiptSchema.parse(parsed), + raw, + rawSha256: sha256(raw), + }; +} + +export async function inputBindings(binaryPath: string, screeningProfilePath: string): Promise<{ + binarySha256: string; + evaluatorSha256: string; + fixtureCorpusSha256: string; + screeningProfileSha256: string; + providerContractSha256: string; +}> { + const [binary, evaluatorSha, profile] = await Promise.all([ + readFile(binaryPath), + evaluatorSourceSha256(), + screeningProfileMetadata(screeningProfilePath), + ]); + return { + binarySha256: sha256(binary), + evaluatorSha256: evaluatorSha, + fixtureCorpusSha256: fixtureCorpusSha256(), + screeningProfileSha256: profile.sha256, + providerContractSha256: profile.providerContractSha256, + }; +} + +export async function assertManifestBoundToInputs( + manifest: CohortManifest, + binaryPath: string, + screeningProfilePath: string, + environment: NodeJS.ProcessEnv = process.env, +): Promise { + const bindings = await inputBindings(binaryPath, screeningProfilePath); + for (const field of [ + "binarySha256", + "evaluatorSha256", + "fixtureCorpusSha256", + "screeningProfileSha256", + "providerContractSha256", + ] as const) { + if (manifest[field] !== bindings[field]) { + throw new Error(`cohort manifest ${field} is not bound to the supplied benchmark input`); + } + } + if (manifest.execution.kind === "github-sigstore-v1") { + if ( + environment.GITHUB_REPOSITORY !== manifest.execution.repository || + environment.GITHUB_SHA !== manifest.execution.sourceSha || + environment.GITHUB_REF !== manifest.execution.sourceRef || + environment.GITHUB_RUN_ID !== manifest.execution.runId || + environment.GITHUB_RUN_ATTEMPT !== manifest.execution.runAttempt + ) { + throw new Error("cohort manifest is not bound to this GitHub Actions source, run, and attempt"); + } + } +} + +export async function createCohortManifest(options: { + purpose: "calibration" | "release"; + binaryPath: string; + screeningProfilePath: string; + runPrefix: string; + execution?: CohortManifest["execution"]; + now?: Date; + uuid?: () => string; +}): Promise { + const count = options.purpose === "calibration" ? 10 : 5; + const uuid = options.uuid ?? randomUUID; + const cohortId = uuid(); + const bindings = await inputBindings(options.binaryPath, options.screeningProfilePath); + const execution = options.execution ?? { + kind: "github-sigstore-v1", + repository: process.env.GITHUB_REPOSITORY ?? "", + signerWorkflow: options.purpose === "release" ? RELEASE_WORKFLOW : CALIBRATION_WORKFLOW, + sourceSha: process.env.GITHUB_SHA ?? "", + sourceRef: process.env.GITHUB_REF ?? "", + runId: process.env.GITHUB_RUN_ID ?? "", + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? "", + } as CohortManifest["execution"]; + return cohortManifestSchema.parse({ + schemaVersion: 2, + purpose: options.purpose, + cohortId, + createdAt: (options.now ?? new Date()).toISOString(), + reportCount: count, + ...bindings, + execution, + slots: Array.from({ length: count }, (_, index) => ({ + slot: index + 1, + runId: `${options.runPrefix}-${String(index + 1).padStart(2, "0")}`, + nonce: uuid(), + })), + }); +} + +function requiredValue(args: readonly string[], index: number, flag: string): string { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${flag} requires a value`); + return value; +} + +async function main(): Promise { + const args = process.argv.slice(2); + let purpose: "calibration" | "release" | undefined; + let binaryPath: string | undefined; + let screeningProfilePath: string | undefined; + let runPrefix: string | undefined; + let outputPath: string | undefined; + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]!; + const value = argument === "--purpose" || argument === "--binary" || + argument === "--screen-profile" || argument === "--run-prefix" || argument === "--out" + ? requiredValue(args, index, argument) + : undefined; + if (argument === "--purpose") { + if (value !== "calibration" && value !== "release") throw new Error("--purpose must be calibration or release"); + purpose = value; + } else if (argument === "--binary") binaryPath = value; + else if (argument === "--screen-profile") screeningProfilePath = value; + else if (argument === "--run-prefix") runPrefix = value; + else if (argument === "--out") outputPath = value; + else throw new Error(`unknown cohort-create argument ${argument}`); + index += 1; + } + if (purpose === undefined || binaryPath === undefined || screeningProfilePath === undefined || + runPrefix === undefined || outputPath === undefined) { + throw new Error("cohort-create requires --purpose, --binary, --screen-profile, --run-prefix, and --out"); + } + const manifest = await createCohortManifest({ + purpose, + binaryPath: resolve(binaryPath), + screeningProfilePath: resolve(screeningProfilePath), + runPrefix, + }); + await writeFile(resolve(outputPath), `${JSON.stringify(manifest, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index f1630bb..6cc77b3 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -4,24 +4,35 @@ import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import { cases } from "../fixtures/cases"; import { - DETECTION_RATE_MAX_DROP_PP, aggregateObservedMetrics, + assertCompleteCohortEvidence, assertDistinctRunIdentities, assertDistinctRawReportDigests, assertExpectedRunIdentities, + assertBaselineCalibrationIntegrity, assertReportsBoundToInputs, assertValidReleaseReport, compareMetrics, + comparisonCohortSha256, + buildCalibrationEvidence, exactMeanCostWithinTolerance, extractObservedMetrics, formatComparisonTable, formatRebaselineGuidance, + isCalibratedBaselineProfile, + meanDetectionCountWithinMargin, median, + parseBaselineFile, parseCliArguments, percentile, type BaselineProfile, type LiveReportForComparison, } from "./compare-baseline"; +import { + sha256, + type CohortManifest, + type CohortReceipt, +} from "./cohort"; import { benchmarkCase } from "./harness"; import { ADMISSION_API_BASE, @@ -37,11 +48,9 @@ import { } from "./livemodels-score"; test("committed baseline authority matches the current benchmark sources", async () => { - const baseline = JSON.parse( + const baseline = parseBaselineFile(JSON.parse( await readFile(resolve(import.meta.dir, "..", "baseline.json"), "utf8"), - ) as { - corpus: { fixtureCorpusSha256: string; evaluatorSha256: string }; - }; + )); const fixtureCorpusSha256 = createHash("sha256") .update(JSON.stringify(cases.map((input) => benchmarkCase.parse(input)))) .digest("hex"); @@ -50,6 +59,26 @@ test("committed baseline authority matches the current benchmark sources", async expect(baseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); }); +test("committed Luna baseline is either fail-closed or has a valid ten-report calibration", async () => { + const baseline = parseBaselineFile(JSON.parse( + await readFile(resolve(import.meta.dir, "..", "baseline.json"), "utf8"), + )); + const profile = baseline.profiles["openai/gpt-5.6-luna"]; + + expect(baseline.schemaVersion).toBe(2); + if (profile === undefined) throw new Error("committed Luna baseline profile must exist"); + if (!profile.populated) { + expect(profile.instructions).toContain("predeclared ten-slot calibration cohort"); + return; + } + expect(isCalibratedBaselineProfile(profile)).toBe(true); + if (!isCalibratedBaselineProfile(profile)) { + throw new Error("committed Luna baseline must contain calibration evidence"); + } + expect(profile.calibration.reportCount).toBe(10); + expect(() => assertBaselineCalibrationIntegrity(profile)).not.toThrow(); +}); + const PROVIDER_CONTRACT: ProviderContractEvidence = { version: 1, benchmarkProviderIdentity: "openrouter:managed-routing", @@ -151,6 +180,7 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { detectionRate: `${detected}/57`, observedProviderCostUsdDecimal, costAccountingComplete: true, + providerGenerationIds: [`gen-fixture-${fakeRunSequence}`], errors: 0, ranAt: options.ranAt ?? new Date(Date.UTC(2026, 7, 25, 0, 0, fakeRunSequence)).toISOString(), }, @@ -220,12 +250,13 @@ const populatedBaseline: Extract = { populated: true, generatedAt: "2026-08-25T00:00:00.000Z", reviewMode: "exhaustive", - sourceRunAt: "2026-08-25T00:00:00.000Z", + sourceRunAt: "2026-08-25T00:00:06.000Z", providerContractEnforced: true, screeningProfileSha256: HASHES.profile, upstreamProviderIdentity: "Azure", totalCases: 70, scoredCases: 70, + defectCases: 57, detectionRate: 54 / 57, falsePositives: 0, gateVerdictCorrectness: 1, @@ -233,8 +264,316 @@ const populatedBaseline: Extract = { maximumRunCostUsdDecimal: "0.07", costCaseCount: 70, latencyMs: { p50: 3500, p95: 6700 }, + calibration: { + reportCount: 10, + cohortId: "00000000-0000-4000-8000-000000000001", + manifestSha256: "9".repeat(64), + sourceSha: "a".repeat(40), + workflowRunId: "456", + binarySha256: HASHES.binary, + providerContractSha256: HASHES.contract, + comparisonCohortSha256: comparisonCohortSha256(fakeReport().summary), + reports: Array.from({ length: 10 }, (_, index) => ({ + slot: index + 1, + nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + runId: `calibration-${index + 1}`, + ranAt: `2026-08-25T00:00:${String(index + 1).padStart(2, "0")}.000Z`, + rawSha256: String(index + 1).padStart(64, "0"), + semanticSha256: String(index + 11).padStart(64, "0"), + receiptRawSha256: String(index + 21).padStart(64, "0"), + binarySha256: HASHES.binary, + detected: 54, + falsePositives: 0, + gateVerdictCorrect: 70, + totalCostUsdDecimal: "0.07", + p50LatencyMs: 3500, + p95LatencyMs: 6700, + })), + }, }; +test("calibration baseline is internally consistent and uses exact count arithmetic", () => { + expect(() => assertBaselineCalibrationIntegrity(populatedBaseline)).not.toThrow(); + expect(meanDetectionCountWithinMargin(540, 10, 260, 5)).toBe(true); + expect(meanDetectionCountWithinMargin(540, 10, 259, 5)).toBe(false); + + const tamperedDetection = structuredClone(populatedBaseline); + tamperedDetection.detectionRate = 53 / 57; + expect(() => assertBaselineCalibrationIntegrity(tamperedDetection)).toThrow( + "detection rate does not match", + ); + + const tamperedLatency = structuredClone(populatedBaseline); + tamperedLatency.latencyMs.p95 += 1; + expect(() => assertBaselineCalibrationIntegrity(tamperedLatency)).toThrow( + "latency does not match", + ); + + const tamperedFalseFindings = structuredClone(populatedBaseline); + tamperedFalseFindings.falsePositives = 1; + expect(() => assertBaselineCalibrationIntegrity(tamperedFalseFindings)).toThrow( + "false finding count does not match", + ); + + const tamperedGateCorrectness = structuredClone(populatedBaseline); + tamperedGateCorrectness.gateVerdictCorrectness = 0.9; + expect(() => assertBaselineCalibrationIntegrity(tamperedGateCorrectness)).toThrow( + "gate verdict correctness does not match", + ); + + const tamperedMaximumCost = structuredClone(populatedBaseline); + tamperedMaximumCost.maximumRunCostUsdDecimal = "0.071"; + expect(() => assertBaselineCalibrationIntegrity(tamperedMaximumCost)).toThrow( + "maximum run cost does not match", + ); + + const tamperedCostCases = structuredClone(populatedBaseline); + tamperedCostCases.costCaseCount = 69; + expect(() => assertBaselineCalibrationIntegrity(tamperedCostCases)).toThrow( + "cost case count does not match", + ); + + const tamperedMeanCost = structuredClone(populatedBaseline); + tamperedMeanCost.meanCostUsdPerCase = 0.002; + expect(() => assertBaselineCalibrationIntegrity(tamperedMeanCost)).toThrow( + "mean cost does not match", + ); + + const tamperedSourceTimestamp = structuredClone(populatedBaseline); + tamperedSourceTimestamp.sourceRunAt = "2026-08-25T00:00:01.000Z"; + expect(() => assertBaselineCalibrationIntegrity(tamperedSourceTimestamp)).toThrow( + "source timestamp does not match", + ); + + const tamperedDetectionBounds = structuredClone(populatedBaseline); + tamperedDetectionBounds.calibration.reports[0]!.detected = 58; + tamperedDetectionBounds.calibration.reports[1]!.detected = 50; + expect(() => assertBaselineCalibrationIntegrity(tamperedDetectionBounds)).toThrow( + "exceeds the defect count", + ); + + const tamperedGateBounds = structuredClone(populatedBaseline); + tamperedGateBounds.calibration.reports[0]!.gateVerdictCorrect = 71; + tamperedGateBounds.calibration.reports[1]!.gateVerdictCorrect = 69; + expect(() => assertBaselineCalibrationIntegrity(tamperedGateBounds)).toThrow( + "exceeds the total case count", + ); + + const tamperedBinaryDigest = structuredClone(populatedBaseline); + tamperedBinaryDigest.calibration.binarySha256 = "a".repeat(64); + expect(() => assertBaselineCalibrationIntegrity(tamperedBinaryDigest)).toThrow( + "different binary digest", + ); + + const fractionalFalseFindingMedian = structuredClone(populatedBaseline); + fractionalFalseFindingMedian.calibration.reports.forEach((report, index) => { + report.falsePositives = index < 5 ? 0 : 1; + }); + fractionalFalseFindingMedian.falsePositives = 0.5; + expect(() => assertBaselineCalibrationIntegrity(fractionalFalseFindingMedian)).not.toThrow(); + const parsed = parseBaselineFile({ + schemaVersion: 2, + corpus: { + fixtureCorpusSha256: HASHES.corpus, + evaluatorSha256: HASHES.evaluator, + }, + profiles: { "openai/gpt-5.6-luna": fractionalFalseFindingMedian }, + }); + expect(parsed.profiles["openai/gpt-5.6-luna"]?.falsePositives).toBe(0.5); +}); + +test("buildCalibrationEvidence preserves ten-run provenance and rejects incomplete identity", () => { + const observed = aggregateObservedMetrics( + Array.from({ length: 10 }, () => fakeReport({ detected: 54 })), + ); + const rawReports = observed.perRun.map((run, index) => ({ + slot: index + 1, + nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + runId: run.runId, + startedAt: `2026-08-25T00:01:${String(index + 1).padStart(2, "0")}.000Z`, + rawSha256: String(index + 1).padStart(64, "0"), + semanticSha256: String(index + 11).padStart(64, "0"), + receiptRawSha256: String(index + 21).padStart(64, "0"), + })); + const manifest = { + cohortId: "00000000-0000-4000-8000-000000000001", + manifestSha256: "9".repeat(64), + sourceSha: "a".repeat(40), + workflowRunId: "456", + }; + const evidence = buildCalibrationEvidence(observed, rawReports, manifest); + expect(evidence.reportCount).toBe(10); + expect(evidence.binarySha256).toBe(HASHES.binary); + expect(evidence.providerContractSha256).toBe(HASHES.contract); + expect(evidence.comparisonCohortSha256).toBe(observed.comparisonCohortSha256); + expect(evidence.reports).toHaveLength(10); + expect(evidence.reports.every((report) => report.detected === 54)).toBe(true); + expect(evidence.reports.every((report) => report.binarySha256 === HASHES.binary)).toBe(true); + expect(evidence.reports.map((report) => report.runId)).toEqual( + observed.perRun.map((run) => run.runId), + ); + expect(evidence.reports.map((report) => report.rawSha256)).toEqual( + rawReports.map((report) => report.rawSha256), + ); + expect(() => buildCalibrationEvidence(observed, rawReports.slice(0, 9), manifest)).toThrow( + "one raw digest per report", + ); + const duplicateRunIds = [...rawReports]; + duplicateRunIds[9] = { ...duplicateRunIds[9]!, runId: duplicateRunIds[0]!.runId }; + expect(() => buildCalibrationEvidence(observed, duplicateRunIds, manifest)).toThrow( + "raw report run IDs must be unique", + ); + const releaseObserved = aggregateObservedMetrics( + Array.from({ length: 5 }, () => fakeReport({ detected: 54 })), + ); + expect(() => buildCalibrationEvidence(releaseObserved, [], manifest)).toThrow( + "requires exactly 10 observed reports", + ); +}); + +function fakeReleaseCohort(reports: readonly LiveReportForComparison[]): { + manifest: CohortManifest; + receipts: Array<{ receipt: CohortReceipt; rawSha256: string }>; + rawReportSha256: string[]; + parsedReports: unknown[]; + expectedRunIds: string[]; +} { + const createdAt = "2026-08-25T00:00:00.000Z"; + const slots = reports.map((report, index) => ({ + slot: index + 1, + runId: report.summary.runId, + nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + })); + const manifest: CohortManifest = { + schemaVersion: 2, + purpose: reports.length === 10 ? "calibration" : "release", + cohortId: "00000000-0000-4000-8000-000000000099", + createdAt, + reportCount: reports.length as 5 | 10, + binarySha256: HASHES.binary, + evaluatorSha256: HASHES.evaluator, + fixtureCorpusSha256: HASHES.corpus, + screeningProfileSha256: HASHES.profile, + providerContractSha256: HASHES.contract, + execution: reports.length === 10 + ? { + kind: "github-sigstore-v1", + repository: "postil-dev/postil-cli", + signerWorkflow: ".github/workflows/benchmark-calibration.yml", + sourceSha: "b".repeat(40), + sourceRef: "refs/heads/main", + runId: "456", + runAttempt: "1", + } + : { + kind: "github-sigstore-v1", + repository: "postil-dev/postil-cli", + signerWorkflow: ".github/workflows/release.yml", + sourceSha: "a".repeat(40), + sourceRef: "refs/tags/v0.0.0", + runId: "123", + runAttempt: "1", + }, + slots, + }; + const parsedReports = reports.map((report) => structuredClone(report)); + const rawReportSha256 = parsedReports.map((report) => sha256(JSON.stringify(report))); + const receipts = reports.map((report, index) => ({ + receipt: { + schemaVersion: 2, + state: "completed", + manifestSha256: "9".repeat(64), + cohortId: manifest.cohortId, + purpose: manifest.purpose, + slot: index + 1, + nonce: slots[index]!.nonce, + runId: report.summary.runId, + startedAt: "2026-08-25T00:00:00.000Z", + finishedAt: "2026-08-25T23:59:59.999Z", + exitCode: 0, + reportRawSha256: rawReportSha256[index]!, + } as CohortReceipt, + rawSha256: String(index + 31).padStart(64, "0"), + })); + return { + manifest, + receipts, + rawReportSha256, + parsedReports, + expectedRunIds: slots.map((slot) => slot.runId), + }; +} + +describe("predeclared cohort evidence", () => { + test("accepts every completed immutable slot", () => { + const reports = Array.from({ length: 5 }, (_, index) => fakeReport({ + durationMultiplier: index + 1, + })); + const cohort = fakeReleaseCohort(reports); + expect(() => assertCompleteCohortEvidence({ + ...cohort, + manifestSha256: "9".repeat(64), + reports, + record: false, + })).not.toThrow(); + }); + + test("allows identical semantic outcomes when every raw artifact is separately authenticated", () => { + const original = fakeReport({ durationMultiplier: 7 }); + const reports = Array.from({ length: 5 }, (_, index) => { + const report = cloneReport(original); + report.summary.runId = `cloned-${index + 1}`; + report.summary.ranAt = `2026-08-25T00:10:0${index}.000Z`; + report.summary.providerGenerationIds = [`gen-cloned-${index + 1}`]; + return report; + }); + const cohort = fakeReleaseCohort(reports); + expect(() => assertCompleteCohortEvidence({ + ...cohort, + manifestSha256: "9".repeat(64), + reports, + record: false, + })).not.toThrow(); + }); + + test("rejects missing, failed, and mismatched slots", () => { + const reports = Array.from({ length: 5 }, (_, index) => fakeReport({ + durationMultiplier: index + 1, + })); + const cohort = fakeReleaseCohort(reports); + expect(() => assertCompleteCohortEvidence({ + ...cohort, + receipts: cohort.receipts.slice(0, 4), + manifestSha256: "9".repeat(64), + reports, + record: false, + })).toThrow("every declared report and receipt slot"); + + const failed = structuredClone(cohort.receipts); + failed[2]!.receipt = { + ...failed[2]!.receipt, + state: "failed", + exitCode: 1, + failure: "benchmark-exit", + } as CohortReceipt; + expect(() => assertCompleteCohortEvidence({ + ...cohort, + receipts: failed, + manifestSha256: "9".repeat(64), + reports, + record: false, + })).toThrow("slot 3 is failed"); + + expect(() => assertCompleteCohortEvidence({ + ...cohort, + expectedRunIds: [...cohort.expectedRunIds.slice(0, 4), "substituted"], + manifestSha256: "9".repeat(64), + reports, + record: false, + })).toThrow("exactly match the predeclared cohort slots"); + }); +}); + describe("sample math", () => { test("uses nearest-rank percentiles", () => { expect(percentile([10, 20, 30, 40, 50], 50)).toBe(30); @@ -267,6 +606,27 @@ describe("sample math", () => { )?.verdict, ).toBe("FAIL"); }); + + test("binds scorer and timeout settings to the calibration execution identity", () => { + const report = fakeReport(); + const scorerChanged = cloneReport(report); + scorerChanged.summary.scorerMode = "enabled"; + scorerChanged.summary.scorerModel = "openai/gpt-5.6-luna"; + const timeoutChanged = cloneReport(report); + timeoutChanged.summary.timeoutOverrides.requestSeconds = "120"; + + expect(comparisonCohortSha256(scorerChanged.summary)).not.toBe( + comparisonCohortSha256(report.summary), + ); + expect(comparisonCohortSha256(timeoutChanged.summary)).not.toBe( + comparisonCohortSha256(report.summary), + ); + const observed = extractObservedMetrics(report); + expect(() => compareMetrics(populatedBaseline, { + ...observed, + comparisonCohortSha256: "f".repeat(64), + })).toThrow("execution identity does not match"); + }); }); describe("release report validation", () => { @@ -407,7 +767,7 @@ describe("release report validation", () => { }); }); -describe("three-report aggregation", () => { +describe("three-report aggregation compatibility", () => { test("52/54/53 passes the detection floor", () => { const observed = aggregateObservedMetrics([ fakeReport({ detected: 52 }), @@ -415,23 +775,20 @@ describe("three-report aggregation", () => { fakeReport({ detected: 53 }), ]); expect(observed.detectionRate).toBe(53 / 57); - expect(populatedBaseline.detectionRate - observed.detectionRate).toBeLessThan( - DETECTION_RATE_MAX_DROP_PP / 100, - ); const comparison = compareMetrics(populatedBaseline, observed); - expect(comparison.rows.find((row) => row.metric === "median detection rate")?.verdict).toBe("PASS"); + expect(comparison.rows.find((row) => row.metric.includes("detection rate"))?.verdict).toBe("PASS"); expect(comparison.ok).toBe(true); }); - test("52/54/52 fails the detection floor", () => { + test("51/52/52 fails the two-defect non-inferiority floor", () => { const observed = aggregateObservedMetrics([ + fakeReport({ detected: 51 }), fakeReport({ detected: 52 }), - fakeReport({ detected: 54 }), fakeReport({ detected: 52 }), ]); - expect(observed.detectionRate).toBe(52 / 57); + expect(observed.detectionRate).toBe(155 / (57 * 3)); const comparison = compareMetrics(populatedBaseline, observed); - expect(comparison.rows.find((row) => row.metric === "median detection rate")?.verdict).toBe("FAIL"); + expect(comparison.rows.find((row) => row.metric.includes("detection rate"))?.verdict).toBe("FAIL"); expect(comparison.ok).toBe(false); }); @@ -458,6 +815,8 @@ describe("three-report aggregation", () => { const first = fakeReport(); const reformatted = cloneReport(first); const rewritten = cloneReport(first); + reformatted.summary.providerGenerationIds = ["gen-reformatted"]; + rewritten.summary.providerGenerationIds = ["gen-rewritten"]; expect(() => assertDistinctRunIdentities([first, reformatted, rewritten])).toThrow( "distinct benchmark run IDs", ); @@ -477,10 +836,10 @@ describe("three-report aggregation", () => { ]); const comparison = compareMetrics(populatedBaseline, observed); const falseFindingRow = comparison.rows.find( - (row) => row.metric === "median false/unrelated findings", + (row) => row.metric.includes("false/unrelated findings"), ); const gateRow = comparison.rows.find( - (row) => row.metric === "median gate verdict correctness", + (row) => row.metric.includes("gate verdict correctness"), ); expect(falseFindingRow?.verdict).toBe("FAIL"); expect(falseFindingRow?.informational).toBe(true); @@ -513,13 +872,75 @@ describe("three-report aggregation", () => { }); }); +describe("five-report release aggregation", () => { + test("uses exact count arithmetic at the two-defect non-inferiority boundary", () => { + const observed = aggregateObservedMetrics([ + fakeReport({ detected: 51 }), + fakeReport({ detected: 52 }), + fakeReport({ detected: 52 }), + fakeReport({ detected: 52 }), + fakeReport({ detected: 53 }), + ]); + expect(observed.reportCount).toBe(5); + expect(observed.detectionRate).toBe(260 / (57 * 5)); + expect(compareMetrics(populatedBaseline, observed).rows.find( + (row) => row.metric.includes("detection rate"), + )?.verdict).toBe("PASS"); + }); + + test("fails when the five-report cohort mean is one defect below the boundary", () => { + const observed = aggregateObservedMetrics([ + fakeReport({ detected: 51 }), + fakeReport({ detected: 52 }), + fakeReport({ detected: 52 }), + fakeReport({ detected: 52 }), + fakeReport({ detected: 52 }), + ]); + expect(observed.detectionRate).toBe(259 / (57 * 5)); + expect(compareMetrics(populatedBaseline, observed).rows.find( + (row) => row.metric.includes("detection rate"), + )?.verdict).toBe("FAIL"); + }); + + test("requires all five reports to share the cohort while retaining full ranges", () => { + const reports = [ + fakeReport({ detected: 49, falsePositives: 0, durationMultiplier: 1 }), + fakeReport({ detected: 50, falsePositives: 1, durationMultiplier: 2 }), + fakeReport({ detected: 51, falsePositives: 0, durationMultiplier: 3 }), + fakeReport({ detected: 52, falsePositives: 2, durationMultiplier: 4 }), + fakeReport({ detected: 53, falsePositives: 1, durationMultiplier: 5 }), + ]; + const observed = aggregateObservedMetrics(reports); + expect(observed.ranges.falsePositives).toEqual({ min: 0, max: 2 }); + expect(observed.ranges.p95LatencyMs).toEqual({ min: 67, max: 335 }); + const mismatched = cloneReport(reports[4]!); + mismatched.summary.binarySha256 = "6".repeat(64); + expect(() => aggregateObservedMetrics([...reports.slice(0, 4), mismatched])).toThrow( + "cohort mismatch for summary.binarySha256", + ); + }); + + test("accepts the ten-report cohort used by baseline recording", () => { + const reports = Array.from({ length: 10 }, () => fakeReport({ detected: 52 })); + const observed = aggregateObservedMetrics(reports); + expect(observed.reportCount).toBe(10); + expect(observed.detectionRate).toBe(52 / 57); + }); +}); + describe("CLI report selection", () => { const releaseInputs = [ "--binary", "target/release/postil", "--screen-profile", "provisional-models.json", ]; + const fiveCohortInputs = [ + "--cohort-manifest", "release-cohort.json", + ...["one", "two", "three", "four", "five"].flatMap((path) => [ + "--receipt", `${path}.receipt.json`, + ]), + ]; - test("accepts exactly one or three distinct --result paths", () => { + test("accepts exactly one, three, or five distinct --result paths", () => { expect(parseCliArguments([ ...releaseInputs, "--expected-run-id", "one", @@ -534,19 +955,39 @@ describe("CLI report selection", () => { "--result", "two.json", "--result", "three.json", ]).resultPaths).toEqual(["one.json", "two.json", "three.json"]); + expect(parseCliArguments([ + ...releaseInputs, + ...["one", "two", "three", "four", "five"].flatMap((runId) => [ + "--expected-run-id", runId, + ]), + ...["one", "two", "three", "four", "five"].flatMap((path) => [ + "--result", `${path}.json`, + ]), + ...fiveCohortInputs, + ]).resultPaths).toEqual([ + "one.json", "two.json", "three.json", "four.json", "five.json", + ]); }); test("rejects every other report count", () => { - expect(() => parseCliArguments([])).toThrow("exactly 1 or 3"); + expect(() => parseCliArguments([])).toThrow("exactly 1, 3, or 5"); expect(() => parseCliArguments(["--result", "one", "--result", "two"])).toThrow( - "exactly 1 or 3", + "exactly 1, 3, or 5", ); expect(() => parseCliArguments([ "--result", "one", "--result", "two", "--result", "three", "--result", "four", - ])).toThrow("exactly 1 or 3"); + ])).toThrow("exactly 1, 3, or 5"); + expect(() => parseCliArguments([ + "--result", "one", + "--result", "two", + "--result", "three", + "--result", "four", + "--result", "five", + "--result", "six", + ])).toThrow("exactly 1, 3, or 5"); }); test("rejects duplicate paths and duplicate raw reports", () => { @@ -566,23 +1007,52 @@ describe("CLI report selection", () => { ])).toThrow("distinct raw SHA-256 digests"); }); - test("requires the same three-sample estimator for rebaselining", () => { + test("requires exactly ten reports for baseline recording", () => { expect(() => parseCliArguments([ ...releaseInputs, "--expected-run-id", "one", "--result", "one.json", "--record", - ])).toThrow("--record requires exactly three --result reports"); + ])).toThrow("--record requires exactly ten --result reports"); expect(parseCliArguments([ ...releaseInputs, "--expected-run-id", "one", "--expected-run-id", "two", "--expected-run-id", "three", + "--expected-run-id", "four", + "--expected-run-id", "five", + "--expected-run-id", "six", + "--expected-run-id", "seven", + "--expected-run-id", "eight", + "--expected-run-id", "nine", + "--expected-run-id", "ten", "--result", "one.json", "--result", "two.json", "--result", "three.json", + "--result", "four.json", + "--result", "five.json", + "--result", "six.json", + "--result", "seven.json", + "--result", "eight.json", + "--result", "nine.json", + "--result", "ten.json", + "--cohort-manifest", "calibration-cohort.json", + ...["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"].flatMap( + (path) => ["--receipt", `${path}.receipt.json`], + ), "--record", ]).record).toBe(true); + expect(() => parseCliArguments([ + ...releaseInputs, + ...["one", "two", "three", "four", "five"].flatMap((runId) => [ + "--expected-run-id", runId, + ]), + ...["one", "two", "three", "four", "five"].flatMap((path) => [ + "--result", `${path}.json`, + ]), + ...fiveCohortInputs, + "--record", + ])).toThrow("--record requires exactly ten --result reports"); }); test("requires explicit release binary and screening profile inputs", () => { @@ -603,22 +1073,60 @@ describe("CLI report selection", () => { ])).toThrow("one --expected-run-id per --result report"); }); - test("prints an executable three-report rebaseline command only for a complete cohort", () => { + test("prints an executable ten-report calibration command only for a complete cohort", () => { expect(formatRebaselineGuidance({ binaryPath: "target/release/postil", screeningProfilePath: "provisional models.json", - expectedRunIds: ["one", "two", "three"], - resultPaths: ["one.json", "two report.json", "three.json"], + expectedRunIds: [ + "one", "two", "three", "four", "five", + "six", "seven", "eight", "nine", "ten", + ], + resultPaths: [ + "one.json", "two report.json", "three.json", "four.json", "five.json", + "six.json", "seven.json", "eight.json", "nine.json", "ten.json", + ], + cohortManifestPath: "calibration cohort.json", + receiptPaths: [ + "one.receipt.json", "two receipt.json", "three.receipt.json", + "four.receipt.json", "five.receipt.json", "six.receipt.json", + "seven.receipt.json", "eight.receipt.json", "nine.receipt.json", + "ten.receipt.json", + ], })).toBe([ " bun run bench:compare -- \\", " --binary 'target/release/postil' \\", " --screen-profile 'provisional models.json' \\", + " --cohort-manifest 'calibration cohort.json' \\", " --expected-run-id 'one' \\", " --expected-run-id 'two' \\", " --expected-run-id 'three' \\", + " --expected-run-id 'four' \\", + " --expected-run-id 'five' \\", + " --expected-run-id 'six' \\", + " --expected-run-id 'seven' \\", + " --expected-run-id 'eight' \\", + " --expected-run-id 'nine' \\", + " --expected-run-id 'ten' \\", " --result 'one.json' \\", " --result 'two report.json' \\", " --result 'three.json' \\", + " --result 'four.json' \\", + " --result 'five.json' \\", + " --result 'six.json' \\", + " --result 'seven.json' \\", + " --result 'eight.json' \\", + " --result 'nine.json' \\", + " --result 'ten.json' \\", + " --receipt 'one.receipt.json' \\", + " --receipt 'two receipt.json' \\", + " --receipt 'three.receipt.json' \\", + " --receipt 'four.receipt.json' \\", + " --receipt 'five.receipt.json' \\", + " --receipt 'six.receipt.json' \\", + " --receipt 'seven.receipt.json' \\", + " --receipt 'eight.receipt.json' \\", + " --receipt 'nine.receipt.json' \\", + " --receipt 'ten.receipt.json' \\", " --record", ].join("\n")); @@ -627,6 +1135,7 @@ describe("CLI report selection", () => { screeningProfilePath: "provisional-models.json", expectedRunIds: ["one"], resultPaths: ["one.json"], - })).toContain("collect three independent complete reports"); + receiptPaths: [], + })).toContain("predeclared ten-report calibration cohort"); }); }); diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index fc0181a..df071a0 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -1,10 +1,11 @@ #!/usr/bin/env bun // Release-gate regression check for the diff-file live benchmark (see live.ts). // -// Consumes one or three LiveReport JSON artifacts written by `bun run -// bench:live --json-out ` and compares their metrics against the committed -// `bench/baseline.json`. Every report must be complete full-corpus evidence. A -// three-report comparison additionally requires one identical benchmark cohort +// Compare mode consumes one, three, or five LiveReport JSON artifacts written by +// `bun run bench:live --json-out ` and compares their metrics against the +// committed `bench/baseline.json`. Record mode requires a predeclared ten-report +// calibration cohort. Every report must be complete full-corpus evidence. A +// multi-report operation additionally requires one identical benchmark cohort // and distinct raw artifacts. Exits non-zero on invalid evidence or a material // regression, so the release pipeline can refuse to ship a CLI that reviews // worse than the last recorded baseline. @@ -14,16 +15,16 @@ // bun run bench:compare -- --binary --screen-profile // --expected-run-id --result // bun run bench:compare -- --binary --screen-profile -// --expected-run-id --expected-run-id --expected-run-id -// --result --result --result +// --expected-run-id ... --expected-run-id +// --result ... --result // -// Record mode writes the three-sample aggregate into baseline.json as the new +// Record mode writes the ten-sample calibration cohort into baseline.json as the new // baseline for the reports' model. This is the deliberate re-baseline path: // nothing updates baseline.json except an explicit --record invocation. // // bun run bench:compare -- --binary --screen-profile -// --expected-run-id --expected-run-id --expected-run-id -// --result --result --result --record +// --expected-run-id ... --expected-run-id +// --result ... --result --record import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; @@ -32,6 +33,14 @@ import { isDeepStrictEqual } from "node:util"; import { z } from "zod"; import { cases } from "../fixtures/cases"; import { benchmarkCase } from "./harness"; +import { + assertManifestBoundToInputs, + readCohortManifest, + readCohortReceipt, + reportSemanticSha256, + type CohortManifest, + type CohortReceipt, +} from "./cohort"; import { ADMISSION_API_BASE, evaluatorSourceSha256, @@ -52,49 +61,21 @@ import { // inference variance (live mode is a single nondeterministic model run per // case) while still catching a real behavioral or cost regression. -/** Detection rate may drop at most this many percentage points below baseline - * before the gate fails. */ -export const DETECTION_RATE_MAX_DROP_PP = 2; +/** The release cohort's mean detections may trail calibration by at most this + * many defect fixtures before the gate fails. */ +export const DETECTION_COUNT_NON_INFERIORITY_MARGIN = 2; /** The false/unrelated finding count above baseline that is reported as a - * concern. Not blocking: see MEASURED_RUN_TO_RUN_SPREAD. An absolute count, - * not a rate, since the corpus size is fixed. */ + * concern. This remains informational because no stable blocking threshold is + * established. An absolute count, not a rate, since the corpus size is fixed. */ export const FALSE_FINDINGS_MAX_INCREASE = 2; /** Gate-verdict correctness (does the CLI's exit code agree with the * authored classification: block must-block, pass everything else) below - * baseline that is reported as a concern. Not blocking: see - * MEASURED_RUN_TO_RUN_SPREAD. */ + * baseline that is reported as a concern. This remains informational because + * no stable blocking threshold is established. */ export const GATE_VERDICT_MAX_DROP_PP = 2; -/** - * Measured run-to-run spread for the informational metrics. - * - * Six runs of a single unchanged binary against this corpus, four on - * OpenRouter managed routing and two pinned to the qualified upstream - * provider, produced: - * - * detection rate 96.5% - 100.0% (3.5pp spread) - * false/unrelated findings 4 - 7 - * gate verdict correctness 71.4% - 84.3% (12.9pp spread) - * - * Every request is issued at temperature 0, so this is not sampling noise. It - * is the provider's own nondeterminism, and pinning the upstream provider did - * not remove it: the widest false-finding count came from a pinned run. - * - * The release gate compares three-run medians for quality and latency, and the - * maximum per-run mean cost. False findings and gate-verdict correctness remain - * informational because this measured sample does not establish sufficiently - * narrow blocking thresholds for those metrics. Their median and complete - * observed range stay visible in the comparison output. - */ -export const MEASURED_RUN_TO_RUN_SPREAD = { - runs: 6, - detectionRatePp: 3.5, - falseFindingsCount: 3, - gateVerdictPp: 12.9, -} as const; - /** Maximum per-run mean provider cost per case may rise at most this fraction * above a baseline recorded under the same enforced provider profile. */ const MEAN_COST_CEILING_NUMERATOR = 5n; @@ -206,6 +187,7 @@ const liveReportSchema = z.object({ detectionRate: nonemptyStringSchema, observedProviderCostUsdDecimal: canonicalCostSchema, costAccountingComplete: z.boolean(), + providerGenerationIds: z.array(z.string().regex(/^gen-[A-Za-z0-9_-]+$/u)).min(1), errors: z.number().int().nonnegative(), ranAt: nonemptyStringSchema, }), @@ -217,6 +199,53 @@ export type LiveReportForComparison = z.infer; // --------------------------------------------------------------------------- // Baseline file shape. +const calibrationReportSchema = z.object({ + slot: z.number().int().positive(), + nonce: z.string().uuid(), + runId: nonemptyStringSchema, + ranAt: nonemptyStringSchema, + rawSha256: sha256Schema, + semanticSha256: sha256Schema, + receiptRawSha256: sha256Schema, + binarySha256: sha256Schema, + detected: z.number().int().nonnegative(), + falsePositives: z.number().int().nonnegative(), + gateVerdictCorrect: z.number().int().nonnegative(), + totalCostUsdDecimal: canonicalCostSchema, + p50LatencyMs: z.number().nonnegative(), + p95LatencyMs: z.number().nonnegative(), +}); + +const calibrationSchema = z.object({ + reportCount: z.literal(10), + cohortId: z.string().uuid(), + manifestSha256: sha256Schema, + sourceSha: z.string().regex(/^[0-9a-f]{40,64}$/u), + workflowRunId: nonemptyStringSchema, + binarySha256: sha256Schema, + providerContractSha256: sha256Schema, + comparisonCohortSha256: sha256Schema, + reports: z.array(calibrationReportSchema).length(10), +}).superRefine((calibration, context) => { + const runIds = calibration.reports.map((report) => report.runId); + if (new Set(runIds).size !== runIds.length) { + context.addIssue({ code: "custom", message: "calibration report run IDs must be unique" }); + } + const rawDigests = calibration.reports.map((report) => report.rawSha256); + if (new Set(rawDigests).size !== rawDigests.length) { + context.addIssue({ code: "custom", message: "calibration report digests must be unique" }); + } + const receiptDigests = calibration.reports.map((report) => report.receiptRawSha256); + if (new Set(receiptDigests).size !== receiptDigests.length) { + context.addIssue({ code: "custom", message: "calibration receipt digests must be unique" }); + } + if (calibration.reports.some((report, index) => report.slot !== index + 1)) { + context.addIssue({ code: "custom", message: "calibration report slots must be complete and ordered" }); + } +}); + +export type CalibrationEvidence = z.infer; + const baselineProfileSchema = z.discriminatedUnion("populated", [ z.object({ populated: z.literal(false), @@ -230,10 +259,12 @@ const baselineProfileSchema = z.discriminatedUnion("populated", [ providerContractEnforced: z.boolean(), screeningProfileSha256: sha256Schema.nullable(), upstreamProviderIdentity: nonemptyStringSchema.nullable(), + calibration: calibrationSchema.optional(), totalCases: z.number().int().positive(), scoredCases: z.number().int().positive(), + defectCases: z.number().int().positive().optional(), detectionRate: z.number().min(0).max(1), - falsePositives: z.number().int().nonnegative(), + falsePositives: z.number().finite().nonnegative(), gateVerdictCorrectness: z.number().min(0).max(1), meanCostUsdPerCase: z.number().nonnegative(), maximumRunCostUsdDecimal: canonicalCostSchema.optional(), @@ -246,7 +277,7 @@ const baselineProfileSchema = z.discriminatedUnion("populated", [ ]); const baselineFileSchema = z.object({ - schemaVersion: z.literal(1), + schemaVersion: z.literal(2), corpus: z.object({ fixtureCorpusSha256: sha256Schema, evaluatorSha256: sha256Schema, @@ -257,21 +288,32 @@ const baselineFileSchema = z.object({ export type BaselineFile = z.infer; export type BaselineProfile = z.infer; +export function parseBaselineFile(value: unknown): BaselineFile { + return baselineFileSchema.parse(value); +} + // --------------------------------------------------------------------------- // Metric extraction from a live report. +type SupportedReportCount = 1 | 3 | 5 | 10; + export interface ObservedMetrics { - reportCount: 1 | 3; + reportCount: SupportedReportCount; model: string; reviewMode: "exhaustive" | "bounded"; providerContractEnforced: boolean; screeningProfileSha256: string | null; upstreamProviderIdentity: string | null; + providerContractSha256: string; + comparisonCohortSha256: string; + binarySha256: string; fixtureCorpusSha256: string; evaluatorSha256: string; ranAt: string; totalCases: number; scoredCases: number; + defectCases: number; + detectedTotal: number; detectionRate: number; falsePositives: number; gateVerdictCorrectness: number; @@ -287,6 +329,16 @@ export interface ObservedMetrics { p50LatencyMs: MetricRange; p95LatencyMs: MetricRange; }; + perRun: Array<{ + runId: string; + ranAt: string; + detected: number; + falsePositives: number; + gateVerdictCorrect: number; + totalCostUsdDecimal: string; + p50LatencyMs: number; + p95LatencyMs: number; + }>; } export interface MetricRange { @@ -419,6 +471,9 @@ export function assertValidReleaseReport(report: LiveReportForComparison): void const ids = report.results.map((result) => result.id); if (new Set(ids).size !== ids.length) invalidReport("result IDs must be unique"); + if (new Set(s.providerGenerationIds).size !== s.providerGenerationIds.length) { + invalidReport("provider generation IDs must be unique"); + } const defectResults = report.results.filter((result) => result.type === "defect"); const cleanResults = report.results.filter((result) => result.type === "clean"); @@ -496,6 +551,17 @@ const COHORT_SUMMARY_FIELDS = [ "scoredCases", ] as const satisfies readonly (keyof LiveReportForComparison["summary"])[]; +export function comparisonCohortSha256( + summary: LiveReportForComparison["summary"], +): string { + const identity = Object.fromEntries( + COHORT_SUMMARY_FIELDS + .filter((field) => field !== "binarySha256") + .map((field) => [field, summary[field]]), + ); + return createHash("sha256").update(JSON.stringify(identity)).digest("hex"); +} + function resultCohort(report: LiveReportForComparison) { return report.results .map(({ id, type, truthSeverity }) => ({ id, type, truthSeverity })) @@ -503,6 +569,10 @@ function resultCohort(report: LiveReportForComparison) { } export function assertMatchingCohort(reports: readonly LiveReportForComparison[]): void { + const providerGenerationIds = reports.flatMap((report) => report.summary.providerGenerationIds); + if (new Set(providerGenerationIds).size !== providerGenerationIds.length) { + throw new Error("live report cohort requires globally distinct provider generation IDs"); + } if (reports.length < 2) return; const reference = reports[0]!; for (let index = 1; index < reports.length; index += 1) { @@ -524,17 +594,21 @@ export function assertDistinctRunIdentities( if (reports.length < 2) return; const runIds = reports.map((report) => report.summary.runId); if (new Set(runIds).size !== runIds.length) { - throw new Error("three-report comparison requires distinct benchmark run IDs"); + throw new Error("multi-report comparison requires distinct benchmark run IDs"); } const runTimes = reports.map((report) => report.summary.ranAt); if (new Set(runTimes).size !== runTimes.length) { - throw new Error("three-report comparison requires distinct benchmark run timestamps"); + throw new Error("multi-report comparison requires distinct benchmark run timestamps"); } } interface PerRunMetrics { + runId: string; + ranAt: string; + detected: number; detectionRate: number; falsePositives: number; + gateVerdictCorrect: number; gateVerdictCorrectness: number; totalCostUsdDecimal: string; meanCostUsdPerCase: number; @@ -553,8 +627,12 @@ function extractPerRunMetrics(report: LiveReportForComparison): PerRunMetrics { .sort((a, b) => a - b); return { + runId: s.runId, + ranAt: s.ranAt, + detected: s.detected, detectionRate: s.detected / s.defectCases, falsePositives: s.falsePositives, + gateVerdictCorrect: gateCorrect, gateVerdictCorrectness: gateCorrect / s.totalCases, totalCostUsdDecimal: s.observedProviderCostUsdDecimal, meanCostUsdPerCase: Number(s.observedProviderCostUsdDecimal) / s.totalCases, @@ -566,14 +644,18 @@ function extractPerRunMetrics(report: LiveReportForComparison): PerRunMetrics { export function aggregateObservedMetrics( reports: readonly LiveReportForComparison[], ): ObservedMetrics { - if (reports.length !== 1 && reports.length !== 3) { - throw new Error(`comparison requires exactly 1 or 3 reports, received ${reports.length}`); + if (reports.length !== 1 && reports.length !== 3 && reports.length !== 5 && reports.length !== 10) { + throw new Error(`aggregation requires exactly 1, 3, 5, or 10 reports, received ${reports.length}`); } reports.forEach(assertValidReleaseReport); assertMatchingCohort(reports); assertDistinctRunIdentities(reports); - const perRun = reports.map(extractPerRunMetrics); + const orderedReports = [...reports].sort((left, right) => + left.summary.runId.localeCompare(right.summary.runId) || + left.summary.ranAt.localeCompare(right.summary.ranAt)); + const perRun = orderedReports.map(extractPerRunMetrics); + const detectedTotal = perRun.reduce((sum, run) => sum + run.detected, 0); const detectionRates = perRun.map((run) => run.detectionRate); const falsePositives = perRun.map((run) => run.falsePositives); const gateCorrectness = perRun.map((run) => run.gateVerdictCorrectness); @@ -589,18 +671,23 @@ export function aggregateObservedMetrics( const ranAt = reports.map((report) => report.summary.ranAt).sort()[Math.floor(reports.length / 2)]!; return { - reportCount: reports.length as 1 | 3, + reportCount: reports.length as SupportedReportCount, model: s.model, reviewMode: s.reviewMode, providerContractEnforced: s.providerContractEnforced, screeningProfileSha256: s.screeningProfileSha256, upstreamProviderIdentity: s.upstreamProviderIdentity, + providerContractSha256: s.providerContractSha256, + comparisonCohortSha256: comparisonCohortSha256(s), + binarySha256: s.binarySha256, fixtureCorpusSha256: s.fixtureCorpusSha256, evaluatorSha256: s.evaluatorSha256, ranAt, totalCases: s.totalCases, scoredCases: s.scoredCases, - detectionRate: median(detectionRates), + defectCases: s.defectCases, + detectedTotal, + detectionRate: detectedTotal / (s.defectCases * reports.length), falsePositives: median(falsePositives), gateVerdictCorrectness: median(gateCorrectness), maximumRunCostUsdDecimal: maximumCostRun.totalCostUsdDecimal, @@ -618,6 +705,16 @@ export function aggregateObservedMetrics( p50LatencyMs: metricRange(p50Latencies), p95LatencyMs: metricRange(p95Latencies), }, + perRun: perRun.map((run) => ({ + runId: run.runId, + ranAt: run.ranAt, + detected: run.detected, + falsePositives: run.falsePositives, + gateVerdictCorrect: run.gateVerdictCorrect, + totalCostUsdDecimal: run.totalCostUsdDecimal, + p50LatencyMs: run.p50LatencyMs, + p95LatencyMs: run.p95LatencyMs, + })), }; } @@ -625,6 +722,55 @@ export function extractObservedMetrics(report: LiveReportForComparison): Observe return aggregateObservedMetrics([report]); } +export interface RawReportProvenance { + slot: number; + nonce: string; + runId: string; + startedAt: string; + rawSha256: string; + semanticSha256: string; + receiptRawSha256: string; +} + +export interface CalibrationManifestProvenance { + cohortId: string; + manifestSha256: string; + sourceSha: string; + workflowRunId: string; +} + +export function buildCalibrationEvidence( + observed: ObservedMetrics, + rawReports: readonly RawReportProvenance[], + manifest: CalibrationManifestProvenance, +): CalibrationEvidence { + if (observed.reportCount !== 10) { + throw new Error(`baseline calibration requires exactly 10 observed reports, received ${observed.reportCount}`); + } + if (rawReports.length !== observed.reportCount) { + throw new Error( + `baseline calibration requires one raw digest per report, received ${rawReports.length}/${observed.reportCount}`, + ); + } + const rawProvenanceByRunId = new Map(rawReports.map((report) => [report.runId, report])); + if (rawProvenanceByRunId.size !== rawReports.length) { + throw new Error("baseline calibration raw report run IDs must be unique"); + } + const metricsByRunId = new Map(observed.perRun.map((run) => [run.runId, run])); + return calibrationSchema.parse({ + reportCount: 10, + ...manifest, + binarySha256: observed.binarySha256, + providerContractSha256: observed.providerContractSha256, + comparisonCohortSha256: observed.comparisonCohortSha256, + reports: [...rawReports].sort((left, right) => left.slot - right.slot).map((provenance) => ({ + ...metricsByRunId.get(provenance.runId), + binarySha256: observed.binarySha256, + ...provenance, + })), + }); +} + // --------------------------------------------------------------------------- // Comparison @@ -634,7 +780,7 @@ interface MetricVerdict { observed: string; verdict: "PASS" | "FAIL"; detail?: string; - /** Reported, but never blocks a release. See MEASURED_RUN_TO_RUN_SPREAD. */ + /** Reported, but never blocks a release. */ informational?: boolean; } @@ -643,6 +789,77 @@ export interface ComparisonResult { rows: MetricVerdict[]; } +export type CalibratedBaselineProfile = Extract & { + calibration: NonNullable["calibration"]>; + defectCases: number; +}; + +export function isCalibratedBaselineProfile( + profile: Extract, +): profile is CalibratedBaselineProfile { + return profile.calibration !== undefined && profile.defectCases !== undefined; +} + +export function assertBaselineCalibrationIntegrity(profile: CalibratedBaselineProfile): void { + const reports = profile.calibration.reports; + const detectedTotal = profile.calibration.reports.reduce( + (sum, report) => sum + report.detected, + 0, + ); + const expectedDetectionRate = detectedTotal / + (profile.calibration.reportCount * profile.defectCases); + if (profile.detectionRate !== expectedDetectionRate) { + throw new Error("baseline detection rate does not match its calibration reports"); + } + for (const report of profile.calibration.reports) { + if (report.binarySha256 !== profile.calibration.binarySha256) { + throw new Error(`baseline calibration report ${report.runId} has a different binary digest`); + } + if (report.detected > profile.defectCases) { + throw new Error(`baseline calibration report ${report.runId} exceeds the defect count`); + } + if (report.gateVerdictCorrect > profile.totalCases) { + throw new Error(`baseline calibration report ${report.runId} exceeds the total case count`); + } + } + const expectedFalsePositives = median(reports.map((report) => report.falsePositives)); + if (profile.falsePositives !== expectedFalsePositives) { + throw new Error("baseline false finding count does not match its calibration reports"); + } + const expectedGateVerdictCorrectness = median( + reports.map((report) => report.gateVerdictCorrect / profile.totalCases), + ); + if (profile.gateVerdictCorrectness !== expectedGateVerdictCorrectness) { + throw new Error("baseline gate verdict correctness does not match its calibration reports"); + } + const maximumCostReport = reports.reduce((maximum, candidate) => + compareCanonicalDecimals( + parseCanonicalDecimal(candidate.totalCostUsdDecimal), + parseCanonicalDecimal(maximum.totalCostUsdDecimal), + ) > 0 ? candidate : maximum); + if (profile.maximumRunCostUsdDecimal !== maximumCostReport.totalCostUsdDecimal) { + throw new Error("baseline maximum run cost does not match its calibration reports"); + } + if (profile.costCaseCount !== profile.totalCases) { + throw new Error("baseline cost case count does not match its complete case count"); + } + const expectedMeanCost = Number(maximumCostReport.totalCostUsdDecimal) / profile.totalCases; + if (profile.meanCostUsdPerCase !== expectedMeanCost) { + throw new Error("baseline mean cost does not match its maximum calibration run"); + } + const expectedP50 = median(reports.map((report) => report.p50LatencyMs)); + const expectedP95 = median(reports.map((report) => report.p95LatencyMs)); + if (profile.latencyMs.p50 !== expectedP50 || profile.latencyMs.p95 !== expectedP95) { + throw new Error("baseline latency does not match its calibration reports"); + } + const expectedSourceRunAt = reports + .map((report) => report.ranAt) + .sort()[Math.floor(reports.length / 2)]!; + if (profile.sourceRunAt !== expectedSourceRunAt) { + throw new Error("baseline source timestamp does not match its calibration reports"); + } +} + function pct(v: number): string { return `${(v * 100).toFixed(1)}%`; } @@ -678,18 +895,59 @@ export function exactMeanCostWithinTolerance( return compareCanonicalDecimals(left, right) <= 0; } -export function compareMetrics(baseline: Extract, observed: ObservedMetrics): ComparisonResult { +export function meanDetectionCountWithinMargin( + baselineDetectedTotal: number, + baselineReportCount: number, + observedDetectedTotal: number, + observedReportCount: number, +): boolean { + for (const [label, value] of [ + ["baseline detected total", baselineDetectedTotal], + ["baseline report count", baselineReportCount], + ["observed detected total", observedDetectedTotal], + ["observed report count", observedReportCount], + ] as const) { + if (!Number.isSafeInteger(value) || value < (label.endsWith("count") ? 1 : 0)) { + throw new Error(`${label} must be a ${label.endsWith("count") ? "positive" : "nonnegative"} safe integer`); + } + } + const baselineTotal = BigInt(baselineDetectedTotal); + const baselineCount = BigInt(baselineReportCount); + const observedTotal = BigInt(observedDetectedTotal); + const observedCount = BigInt(observedReportCount); + const margin = BigInt(DETECTION_COUNT_NON_INFERIORITY_MARGIN); + return observedTotal * baselineCount + margin * observedCount * baselineCount >= + baselineTotal * observedCount; +} + +export function compareMetrics(baseline: CalibratedBaselineProfile, observed: ObservedMetrics): ComparisonResult { + if (baseline.calibration.comparisonCohortSha256 !== observed.comparisonCohortSha256) { + throw new Error("baseline calibration execution identity does not match the candidate cohort"); + } const rows: MetricVerdict[] = []; const sampleLabel = observed.reportCount === 1 ? "1 run" : `${observed.reportCount} runs`; - const detectionFloor = baseline.detectionRate - DETECTION_RATE_MAX_DROP_PP / 100; + const calibrationDetectedTotal = baseline.calibration.reports.reduce( + (sum, report) => sum + report.detected, + 0, + ); + const detectionFloor = Math.max( + 0, + baseline.detectionRate - DETECTION_COUNT_NON_INFERIORITY_MARGIN / observed.defectCases, + ); + const detectionWithinMargin = meanDetectionCountWithinMargin( + calibrationDetectedTotal, + baseline.calibration.reportCount, + observed.detectedTotal, + observed.reportCount, + ); rows.push({ - metric: "median detection rate", + metric: "mean detection rate", baseline: pct(baseline.detectionRate), observed: pct(observed.detectionRate), - verdict: observed.detectionRate >= detectionFloor ? "PASS" : "FAIL", + verdict: detectionWithinMargin ? "PASS" : "FAIL", detail: - `floor ${pct(detectionFloor)} (baseline - ${DETECTION_RATE_MAX_DROP_PP}pp); ` + + `floor ${pct(detectionFloor)} (baseline mean - ${DETECTION_COUNT_NON_INFERIORITY_MARGIN} defect fixtures); ` + `${sampleLabel} range ${pct(observed.ranges.detectionRate.min)}-${pct(observed.ranges.detectionRate.max)}`, }); @@ -811,6 +1069,8 @@ export interface CompareCliOptions { baselinePath: string; binaryPath: string; screeningProfilePath: string; + cohortManifestPath?: string; + receiptPaths: string[]; expectedRunIds: string[]; resultPaths: string[]; record: boolean; @@ -837,10 +1097,12 @@ export function assertDistinctResultPaths(paths: readonly string[]): void { export function parseCliArguments(args: readonly string[]): CompareCliOptions { const resultPaths: string[] = []; + const receiptPaths: string[] = []; const expectedRunIds: string[] = []; let baselinePath = defaultBaselinePath(); let binaryPath: string | undefined; let screeningProfilePath: string | undefined; + let cohortManifestPath: string | undefined; let baselineSeen = false; let record = false; @@ -851,6 +1113,17 @@ export function parseCliArguments(args: readonly string[]): CompareCliOptions { index += 1; continue; } + if (argument === "--receipt") { + receiptPaths.push(requiredFlagValue(args, index, argument)); + index += 1; + continue; + } + if (argument === "--cohort-manifest") { + if (cohortManifestPath !== undefined) throw new Error("--cohort-manifest may be specified only once"); + cohortManifestPath = requiredFlagValue(args, index, argument); + index += 1; + continue; + } if (argument === "--baseline") { if (baselineSeen) throw new Error("--baseline may be specified only once"); baselinePath = requiredFlagValue(args, index, argument); @@ -885,11 +1158,16 @@ export function parseCliArguments(args: readonly string[]): CompareCliOptions { throw new Error(`unknown bench:compare argument ${argument}`); } - if (record && resultPaths.length !== 3) { - throw new Error(`--record requires exactly three --result reports, received ${resultPaths.length}`); + if (record && resultPaths.length !== 10) { + throw new Error(`--record requires exactly ten --result reports, received ${resultPaths.length}`); } - if (!record && resultPaths.length !== 1 && resultPaths.length !== 3) { - throw new Error(`bench:compare requires exactly 1 or 3 --result reports, received ${resultPaths.length}`); + if ( + !record && + resultPaths.length !== 1 && + resultPaths.length !== 3 && + resultPaths.length !== 5 + ) { + throw new Error(`bench:compare requires exactly 1, 3, or 5 --result reports, received ${resultPaths.length}`); } if (binaryPath === undefined) throw new Error("bench:compare requires --binary"); if (screeningProfilePath === undefined) { @@ -904,10 +1182,25 @@ export function parseCliArguments(args: readonly string[]): CompareCliOptions { throw new Error("every --expected-run-id must be distinct"); } assertDistinctResultPaths(resultPaths); + assertDistinctResultPaths(receiptPaths); + const requiresCohort = record || resultPaths.length === 5; + if (requiresCohort && cohortManifestPath === undefined) { + throw new Error("five-report release and ten-report record operations require --cohort-manifest"); + } + if (requiresCohort && receiptPaths.length !== resultPaths.length) { + throw new Error( + `cohort comparison requires one --receipt per --result report, received ${receiptPaths.length}/${resultPaths.length}`, + ); + } + if (!requiresCohort && (cohortManifestPath !== undefined || receiptPaths.length !== 0)) { + throw new Error("cohort evidence is accepted only for five-report release or ten-report record operations"); + } return { baselinePath, binaryPath, screeningProfilePath, + cohortManifestPath, + receiptPaths, expectedRunIds, resultPaths, record, @@ -916,7 +1209,7 @@ export function parseCliArguments(args: readonly string[]): CompareCliOptions { export function assertDistinctRawReportDigests(digests: readonly string[]): void { if (new Set(digests).size !== digests.length) { - throw new Error("three-report comparison requires distinct raw SHA-256 digests"); + throw new Error("multi-report comparison requires distinct raw SHA-256 digests"); } } @@ -936,7 +1229,14 @@ export function assertExpectedRunIdentities( } } -async function loadReports(paths: readonly string[]): Promise { +interface LoadedReports { + reports: LiveReportForComparison[]; + raw: Uint8Array[]; + rawSha256: string[]; + parsed: unknown[]; +} + +async function loadReports(paths: readonly string[]): Promise { const rawReports = await Promise.all(paths.map(async (path) => { const raw = await readFile(path).catch((error) => { throw new Error(`could not read live report at ${path}: ${error instanceof Error ? error.message : String(error)}`); @@ -947,11 +1247,11 @@ async function loadReports(paths: readonly string[]): Promise 1) { assertDistinctRawReportDigests(rawReports.map((report) => report.sha256)); } - return rawReports.map(({ path, raw }) => { + const reports = rawReports.map(({ path, raw }) => { let parsed: unknown; try { parsed = JSON.parse(raw.toString("utf8")); @@ -969,6 +1269,114 @@ async function loadReports(paths: readonly string[]): Promise report.raw), + rawSha256: rawReports.map((report) => report.sha256), + parsed: rawReports.map((report) => JSON.parse(report.raw.toString("utf8")) as unknown), + }; +} + +export interface VerifiedCohortEvidence { + manifest: CohortManifest; + manifestSha256: string; + reports: RawReportProvenance[]; +} + +export function assertCompleteCohortEvidence(options: { + manifest: CohortManifest; + manifestSha256: string; + receipts: Array<{ receipt: CohortReceipt; rawSha256: string }>; + reports: readonly LiveReportForComparison[]; + rawReportSha256: readonly string[]; + parsedReports: readonly unknown[]; + expectedRunIds: readonly string[]; + record: boolean; +}): VerifiedCohortEvidence { + const { manifest } = options; + const expectedPurpose = options.record ? "calibration" : "release"; + if (manifest.purpose !== expectedPurpose) { + throw new Error(`benchmark cohort purpose must be ${expectedPurpose}`); + } + if ( + options.receipts.length !== manifest.reportCount || + options.reports.length !== manifest.reportCount || + options.rawReportSha256.length !== manifest.reportCount || + options.parsedReports.length !== manifest.reportCount + ) { + throw new Error("benchmark cohort requires every declared report and receipt slot"); + } + const manifestRunIds = manifest.slots.map((slot) => slot.runId); + if (!isDeepStrictEqual(options.expectedRunIds, manifestRunIds)) { + throw new Error("expected run IDs must exactly match the predeclared cohort slots"); + } + const receiptBySlot = new Map(options.receipts.map((entry) => [entry.receipt.slot, entry])); + if (receiptBySlot.size !== options.receipts.length) { + throw new Error("benchmark cohort receipt slots must be unique"); + } + + const provenance = manifest.slots.map((slot, index): RawReportProvenance => { + const entry = receiptBySlot.get(slot.slot); + if (entry === undefined) throw new Error(`benchmark cohort is missing receipt slot ${slot.slot}`); + const { receipt } = entry; + if (receipt.state !== "completed") { + throw new Error(`benchmark cohort slot ${slot.slot} is ${receipt.state}; the whole cohort is invalid`); + } + const bindings: Array<[string, unknown, unknown]> = [ + ["manifestSha256", receipt.manifestSha256, options.manifestSha256], + ["cohortId", receipt.cohortId, manifest.cohortId], + ["purpose", receipt.purpose, manifest.purpose], + ["slot", receipt.slot, slot.slot], + ["nonce", receipt.nonce, slot.nonce], + ["runId", receipt.runId, slot.runId], + ]; + for (const [field, actual, expected] of bindings) { + if (actual !== expected) { + throw new Error(`benchmark cohort receipt slot ${slot.slot} ${field} does not match its manifest`); + } + } + const report = options.reports[index]!; + if (report.summary.runId !== slot.runId) { + throw new Error(`benchmark report slot ${slot.slot} does not match its predeclared run ID`); + } + if (receipt.reportRawSha256 !== options.rawReportSha256[index]) { + throw new Error(`benchmark report slot ${slot.slot} raw digest does not match its receipt`); + } + const semanticSha256 = reportSemanticSha256(options.parsedReports[index]); + if (Date.parse(receipt.startedAt) < Date.parse(manifest.createdAt)) { + throw new Error(`benchmark cohort slot ${slot.slot} started before its manifest was created`); + } + if (Date.parse(receipt.finishedAt) < Date.parse(receipt.startedAt)) { + throw new Error(`benchmark cohort slot ${slot.slot} finished before it started`); + } + const reportRanAt = Date.parse(report.summary.ranAt); + if ( + !Number.isFinite(reportRanAt) || + reportRanAt < Date.parse(receipt.startedAt) || + reportRanAt > Date.parse(receipt.finishedAt) + ) { + throw new Error(`benchmark report slot ${slot.slot} ranAt is outside its receipt interval`); + } + return { + slot: slot.slot, + nonce: slot.nonce, + runId: slot.runId, + startedAt: receipt.startedAt, + rawSha256: options.rawReportSha256[index]!, + semanticSha256, + receiptRawSha256: entry.rawSha256, + }; + }); + + for (const [label, values] of [ + ["receipt", provenance.map((report) => report.receiptRawSha256)], + ["raw report", provenance.map((report) => report.rawSha256)], + ] as const) { + if (new Set(values).size !== values.length) { + throw new Error(`benchmark cohort requires distinct ${label} SHA-256 digests`); + } + } + return { manifest, manifestSha256: options.manifestSha256, reports: provenance }; } export async function assertReportsBoundToInputs( @@ -1039,18 +1447,24 @@ function shellQuote(value: string): string { export function formatRebaselineGuidance(options: Pick< CompareCliOptions, - "binaryPath" | "screeningProfilePath" | "expectedRunIds" | "resultPaths" + "binaryPath" | "screeningProfilePath" | "cohortManifestPath" | "receiptPaths" | + "expectedRunIds" | "resultPaths" >): string { - if (options.resultPaths.length !== 3 || options.expectedRunIds.length !== 3) { - return " collect three independent complete reports, then use the three-report --record command documented in bench/README.md"; + if ( + options.resultPaths.length !== 10 || options.expectedRunIds.length !== 10 || + options.receiptPaths.length !== 10 || options.cohortManifestPath === undefined + ) { + return " collect a predeclared ten-report calibration cohort, then use the ten-report --record command documented in bench/README.md"; } return [ " bun run bench:compare -- \\", ` --binary ${shellQuote(options.binaryPath)} \\`, ` --screen-profile ${shellQuote(options.screeningProfilePath)} \\`, + ` --cohort-manifest ${shellQuote(options.cohortManifestPath)} \\`, ...options.expectedRunIds.map((runId) => ` --expected-run-id ${shellQuote(runId)} \\`), ...options.resultPaths.map((path) => ` --result ${shellQuote(path)} \\`), + ...options.receiptPaths.map((path) => ` --receipt ${shellQuote(path)} \\`), " --record", ].join("\n"); } @@ -1060,17 +1474,48 @@ async function main() { baselinePath, binaryPath, screeningProfilePath, + cohortManifestPath, + receiptPaths, expectedRunIds, resultPaths, record, } = parseCliArguments(process.argv.slice(2)); - const reports = await loadReports(resultPaths); + const loadedReports = await loadReports(resultPaths); + const { reports, rawSha256 } = loadedReports; assertExpectedRunIdentities(reports, expectedRunIds); await assertReportsBoundToInputs(reports, binaryPath, screeningProfilePath); + let cohortEvidence: VerifiedCohortEvidence | undefined; + if (cohortManifestPath !== undefined) { + const manifestFile = await readCohortManifest(cohortManifestPath); + await assertManifestBoundToInputs( + manifestFile.manifest, + binaryPath, + screeningProfilePath, + ); + const receipts = await Promise.all(receiptPaths.map(async (path) => { + try { + return await readCohortReceipt(path); + } catch (error) { + throw new Error(`could not read cohort receipt at ${path}: ${error instanceof Error ? error.message : String(error)}`); + } + })); + cohortEvidence = assertCompleteCohortEvidence({ + manifest: manifestFile.manifest, + manifestSha256: manifestFile.rawSha256, + receipts: receipts.map((receipt) => ({ receipt: receipt.receipt, rawSha256: receipt.rawSha256 })), + reports, + rawReportSha256: rawSha256, + parsedReports: loadedReports.parsed, + expectedRunIds, + record, + }); + } const observed = aggregateObservedMetrics(reports); const rebaselineGuidance = formatRebaselineGuidance({ binaryPath, screeningProfilePath, + cohortManifestPath, + receiptPaths, expectedRunIds, resultPaths, }); @@ -1078,9 +1523,22 @@ async function main() { const baselineRaw = await readJson(baselinePath).catch((error) => { throw new Error(`could not read baseline at ${baselinePath}: ${error instanceof Error ? error.message : String(error)}`); }); - const baselineFile = baselineFileSchema.parse(baselineRaw); + const baselineFile = parseBaselineFile(baselineRaw); if (record) { + if (cohortEvidence === undefined || cohortManifestPath === undefined) { + throw new Error("baseline recording requires verified pre-execution cohort evidence"); + } + const calibration = buildCalibrationEvidence( + observed, + cohortEvidence.reports, + { + cohortId: cohortEvidence.manifest.cohortId, + manifestSha256: cohortEvidence.manifestSha256, + sourceSha: cohortEvidence.manifest.execution.sourceSha, + workflowRunId: cohortEvidence.manifest.execution.runId, + }, + ); baselineFile.profiles[observed.model] = { populated: true, generatedAt: new Date().toISOString(), @@ -1089,8 +1547,10 @@ async function main() { providerContractEnforced: observed.providerContractEnforced, screeningProfileSha256: observed.screeningProfileSha256, upstreamProviderIdentity: observed.upstreamProviderIdentity, + calibration, totalCases: observed.totalCases, scoredCases: observed.scoredCases, + defectCases: observed.defectCases, detectionRate: observed.detectionRate, falsePositives: observed.falsePositives, gateVerdictCorrectness: observed.gateVerdictCorrectness, @@ -1144,10 +1604,29 @@ async function main() { return; } + if (!isCalibratedBaselineProfile(profile)) { + console.error( + `The baseline for ${observed.model} lacks the required ten-report calibration evidence.\n` + + rebaselineGuidance, + ); + process.exitCode = 1; + return; + } + try { + assertBaselineCalibrationIntegrity(profile); + } catch (error) { + console.error( + `INVALID BASELINE CALIBRATION: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + return; + } + if ( !profile.providerContractEnforced || profile.screeningProfileSha256 !== observed.screeningProfileSha256 || profile.upstreamProviderIdentity !== observed.upstreamProviderIdentity || + profile.calibration.providerContractSha256 !== observed.providerContractSha256 || profile.maximumRunCostUsdDecimal === undefined || profile.costCaseCount === undefined || profile.costCaseCount !== profile.totalCases @@ -1163,9 +1642,18 @@ async function main() { return; } + if (profile.calibration.comparisonCohortSha256 !== observed.comparisonCohortSha256) { + console.error( + "BASELINE EXECUTION MISMATCH: the candidate scorer, timeout, provider, or other execution identity differs from calibration.", + ); + process.exitCode = 1; + return; + } + if ( profile.reviewMode !== observed.reviewMode || profile.totalCases !== observed.totalCases || + profile.defectCases !== observed.defectCases || profile.scoredCases !== profile.totalCases || profile.scoredCases !== observed.scoredCases ) { @@ -1173,6 +1661,7 @@ async function main() { "BASELINE COHORT MISMATCH: the validated report does not match the baseline execution mode or complete case count.\n" + ` baseline reviewMode ${profile.reviewMode}; observed ${observed.reviewMode}\n` + ` baseline totalCases ${profile.totalCases}; observed ${observed.totalCases}\n` + + ` baseline defectCases ${profile.defectCases}; observed ${observed.defectCases}\n` + ` baseline scoredCases ${profile.scoredCases}; observed ${observed.scoredCases}`, ); process.exitCode = 1; diff --git a/bench/src/generation-evidence.test.ts b/bench/src/generation-evidence.test.ts new file mode 100644 index 0000000..eab008f --- /dev/null +++ b/bench/src/generation-evidence.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, test } from "bun:test"; +import { verifyGenerationEvidence } from "./generation-evidence"; +import { sha256 } from "./cohort"; + +function sample( + ids: string[], + overrides: Record = {}, + receiptOverrides: Record = {}, +) { + const report = { + summary: { + runId: "fixture-run", + ranAt: "2026-08-26T12:00:30.000Z", + model: "openai/gpt-5.6-luna", + upstreamProviderIdentity: "Azure", + totalTokens: { prompt: 30, completion: 12, total: 42 }, + observedProviderCostUsdDecimal: "0.0042", + providerGenerationIds: ids, + ...overrides, + }, + }; + const reportRawSha256 = sha256(JSON.stringify(report)); + return { + report, + reportRawSha256, + receipt: { + schemaVersion: 2, + manifestSha256: "a".repeat(64), + cohortId: "2ad7f189-b050-4182-8935-7bb3df949dfb", + purpose: "release", + slot: 1, + nonce: "6c2908e2-5a3c-491e-87d5-79df0d13ba6c", + runId: "fixture-run", + startedAt: "2026-08-26T12:00:00.000Z", + state: "completed", + finishedAt: "2026-08-26T12:01:00.000Z", + exitCode: 0, + reportRawSha256, + ...receiptOverrides, + }, + }; +} + +function generationFetch(records: Record>): typeof fetch { + return (async (input: string | URL | Request) => { + const id = new URL(input instanceof Request ? input.url : input.toString()).searchParams.get("id")!; + const data = records[id]; + return data === undefined + ? Response.json({ error: "missing" }, { status: 404 }) + : Response.json({ data }); + }) as typeof fetch; +} + +const records = { + "gen-one": { + id: "gen-one", + created_at: "2026-08-26T12:00:10.000Z", + model: "openai/gpt-5.6-luna", + provider_name: "Azure", + tokens_prompt: 10, + tokens_completion: 5, + total_cost: 0.0015, + }, + "gen-two": { + id: "gen-two", + created_at: "2026-08-26T12:00:20.000Z", + model: "openai/gpt-5.6-luna", + provider_name: "Azure", + tokens_prompt: 20, + tokens_completion: 7, + total_cost: 0.0027, + }, +}; + +describe("provider generation evidence", () => { + test("verifies distinct generation identity, route, tokens, and cost", async () => { + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + fetchImpl: generationFetch(records), + })).resolves.toBe(2); + }); + + test("rejects a generation reused across cohort reports", async () => { + await expect(verifyGenerationEvidence([ + sample(["gen-one", "gen-two"]), + sample(["gen-one", "gen-two"]), + ], { + apiKey: "fixture", + fetchImpl: generationFetch(records), + })).rejects.toThrow("duplicate provider generation IDs"); + }); + + test("rejects mismatched provider metadata and accounting", async () => { + await expect(verifyGenerationEvidence([ + sample(["gen-one", "gen-two"], { totalTokens: { prompt: 31, completion: 12, total: 43 } }), + ], { + apiKey: "fixture", + fetchImpl: generationFetch(records), + })).rejects.toThrow("token totals do not match provider generations"); + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + fetchImpl: generationFetch({ + ...records, + "gen-two": { ...records["gen-two"], provider_name: "Other" }, + }), + })).rejects.toThrow("generation from another provider"); + }); + + test("rejects a lookup whose returned generation identity differs", async () => { + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + fetchImpl: generationFetch({ + ...records, + "gen-two": { ...records["gen-two"], id: "gen-one" }, + }), + })).rejects.toThrow("generation identity does not match its lookup"); + }); + + test("binds every generation to the attested receipt interval", async () => { + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + fetchImpl: generationFetch({ + ...records, + "gen-two": { ...records["gen-two"], created_at: "2024-01-01T00:00:00.000Z" }, + }), + })).rejects.toThrow("generation outside its receipt interval"); + }); + + test("requires the exact report and run identity bound by the receipt", async () => { + await expect(verifyGenerationEvidence([ + sample(["gen-one", "gen-two"], {}, { reportRawSha256: "b".repeat(64) }), + ], { + apiKey: "fixture", + fetchImpl: generationFetch(records), + })).rejects.toThrow("does not match its receipt digest"); + + await expect(verifyGenerationEvidence([ + sample(["gen-one", "gen-two"], {}, { runId: "another-run" }), + ], { + apiKey: "fixture", + fetchImpl: generationFetch(records), + })).rejects.toThrow("does not match its receipt run identity"); + }); +}); diff --git a/bench/src/generation-evidence.ts b/bench/src/generation-evidence.ts new file mode 100644 index 0000000..50e5906 --- /dev/null +++ b/bench/src/generation-evidence.ts @@ -0,0 +1,241 @@ +#!/usr/bin/env bun +// Verifies release benchmark generation identities against OpenRouter's audit API. + +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { z } from "zod"; +import { API_KEY_ENV_NAMES_TEXT, resolveApiKeyName } from "./api-key"; +import { cohortReceiptSchema, sha256 } from "./cohort"; + +const GENERATION_API = "https://openrouter.ai/api/v1/generation"; +const MAX_ATTEMPTS = 5; +const MAX_RETRY_MS = 5_000; +const generationIdSchema = z.string().regex(/^gen-[A-Za-z0-9_-]+$/u); + +const reportSchema = z.object({ + summary: z.object({ + runId: z.string().trim().min(1), + ranAt: z.string().datetime({ offset: true }), + model: z.string().trim().min(1), + upstreamProviderIdentity: z.string().trim().min(1), + totalTokens: z.object({ + prompt: z.number().int().nonnegative(), + completion: z.number().int().nonnegative(), + total: z.number().int().nonnegative(), + }), + observedProviderCostUsdDecimal: z.string().regex(/^(?:0|[1-9][0-9]*|(?:0|[1-9][0-9]*)\.[0-9]*[1-9])$/u), + providerGenerationIds: z.array(generationIdSchema).min(1), + }), +}); + +const generationSchema = z.object({ + data: z.object({ + id: generationIdSchema, + created_at: z.string().datetime({ offset: true }), + model: z.string().trim().min(1), + provider_name: z.string().trim().min(1), + tokens_prompt: z.number().int().nonnegative(), + tokens_completion: z.number().int().nonnegative(), + total_cost: z.number().finite().nonnegative(), + }), +}); + +type Report = z.infer; +type Generation = z.infer["data"]; + +export interface GenerationEvidenceSample { + report: unknown; + receipt: unknown; + reportRawSha256: string; +} + +function retryDelay(response: Response, attempt: number): number { + const retryAfter = response.headers.get("retry-after")?.trim(); + if (retryAfter !== undefined && /^\d+$/u.test(retryAfter)) { + return Math.min(MAX_RETRY_MS, Number(retryAfter) * 1_000); + } + return Math.min(MAX_RETRY_MS, 250 * 2 ** attempt); +} + +async function fetchGeneration( + generationId: string, + apiKey: string, + fetchImpl: typeof fetch, +): Promise { + const url = new URL(GENERATION_API); + url.searchParams.set("id", generationId); + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) { + let response: Response; + try { + response = await fetchImpl(url, { + headers: { Authorization: `Bearer ${apiKey}` }, + redirect: "error", + signal: AbortSignal.timeout(15_000), + }); + } catch (error) { + if (attempt + 1 === MAX_ATTEMPTS) { + throw new Error(`generation evidence lookup failed after ${MAX_ATTEMPTS} attempts: ${ + error instanceof Error ? error.message : String(error) + }`); + } + await Bun.sleep(Math.min(MAX_RETRY_MS, 250 * 2 ** attempt)); + continue; + } + if (response.ok) { + const raw = await response.text(); + if (Buffer.byteLength(raw) > 64 * 1024) { + throw new Error("generation evidence response exceeds 64 KiB"); + } + return generationSchema.parse(JSON.parse(raw)).data; + } + if (![404, 429, 500, 502, 503, 504].includes(response.status) || attempt + 1 === MAX_ATTEMPTS) { + throw new Error(`generation evidence lookup returned HTTP ${response.status}`); + } + await Bun.sleep(retryDelay(response, attempt)); + } + throw new Error("generation evidence lookup exhausted its retry budget"); +} + +export async function verifyGenerationEvidence( + samples: readonly GenerationEvidenceSample[], + options: { apiKey: string; fetchImpl?: typeof fetch; concurrency?: number }, +): Promise { + const parsed = samples.map((sample, sampleIndex) => { + const report = reportSchema.parse(sample.report); + const receipt = cohortReceiptSchema.parse(sample.receipt); + if (receipt.state !== "completed") { + throw new Error(`benchmark report ${sampleIndex + 1} does not have a completed receipt`); + } + if (sample.reportRawSha256 !== receipt.reportRawSha256) { + throw new Error(`benchmark report ${sampleIndex + 1} does not match its receipt digest`); + } + if (report.summary.runId !== receipt.runId) { + throw new Error(`benchmark report ${sampleIndex + 1} does not match its receipt run identity`); + } + const startedAt = Date.parse(receipt.startedAt); + const finishedAt = Date.parse(receipt.finishedAt); + const ranAt = Date.parse(report.summary.ranAt); + if (ranAt < startedAt || ranAt > finishedAt) { + throw new Error(`benchmark report ${sampleIndex + 1} timestamp is outside its receipt interval`); + } + return { report, receipt, startedAt, finishedAt }; + }); + const expected = parsed.flatMap((report, reportIndex) => + report.report.summary.providerGenerationIds.map((generationId) => ({ generationId, reportIndex })) + ); + if (new Set(expected.map(({ generationId }) => generationId)).size !== expected.length) { + throw new Error("benchmark cohort contains duplicate provider generation IDs"); + } + const generations = new Array(expected.length); + const concurrency = Math.max(1, Math.min(options.concurrency ?? 4, expected.length)); + let cursor = 0; + await Promise.all(Array.from({ length: concurrency }, async () => { + for (;;) { + const index = cursor++; + if (index >= expected.length) return; + generations[index] = await fetchGeneration( + expected[index]!.generationId, + options.apiKey, + options.fetchImpl ?? fetch, + ); + } + })); + + for (const [generationIndex, generation] of generations.entries()) { + const expectation = expected[generationIndex]!; + const sample = parsed[expectation.reportIndex]!; + if (generation.id !== expectation.generationId) { + throw new Error(`benchmark report ${expectation.reportIndex + 1} generation identity does not match its lookup`); + } + const createdAt = Date.parse(generation.created_at); + if (createdAt < sample.startedAt || createdAt > sample.finishedAt) { + throw new Error( + `benchmark report ${expectation.reportIndex + 1} contains a generation outside its receipt interval`, + ); + } + } + + for (const [reportIndex, sample] of parsed.entries()) { + const report = sample.report; + const reportGenerations = generations.filter((_, generationIndex) => + expected[generationIndex]!.reportIndex === reportIndex + ); + if (reportGenerations.some((generation) => generation.model !== report.summary.model)) { + throw new Error(`benchmark report ${reportIndex + 1} contains a generation for another model`); + } + if (reportGenerations.some((generation) => + generation.provider_name !== report.summary.upstreamProviderIdentity + )) { + throw new Error(`benchmark report ${reportIndex + 1} contains a generation from another provider`); + } + const promptTokens = reportGenerations.reduce((sum, generation) => sum + generation.tokens_prompt, 0); + const completionTokens = reportGenerations.reduce( + (sum, generation) => sum + generation.tokens_completion, + 0, + ); + if (promptTokens !== report.summary.totalTokens.prompt || + completionTokens !== report.summary.totalTokens.completion || + promptTokens + completionTokens !== report.summary.totalTokens.total) { + throw new Error(`benchmark report ${reportIndex + 1} token totals do not match provider generations`); + } + const providerCost = reportGenerations.reduce((sum, generation) => sum + generation.total_cost, 0); + const reportCost = Number(report.summary.observedProviderCostUsdDecimal); + if (!Number.isFinite(reportCost) || Math.abs(providerCost - reportCost) > 1e-9) { + throw new Error(`benchmark report ${reportIndex + 1} cost does not match provider generations`); + } + } + return generations.length; +} + +function requiredValue(args: readonly string[], index: number, flag: string): string { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${flag} requires a value`); + return value; +} + +async function main(): Promise { + const args = process.argv.slice(2); + const samplePaths: Array<{ reportPath: string; receiptPath: string }> = []; + for (let index = 0; index < args.length; index += 4) { + const resultFlag = args[index]; + const receiptFlag = args[index + 2]; + if (resultFlag !== "--result") { + throw new Error(`expected --result, received ${resultFlag ?? "end of arguments"}`); + } + if (receiptFlag !== "--receipt") { + throw new Error(`each --result must be followed by --receipt`); + } + samplePaths.push({ + reportPath: resolve(requiredValue(args, index, resultFlag)), + receiptPath: resolve(requiredValue(args, index + 2, receiptFlag)), + }); + } + if (samplePaths.length === 0) { + throw new Error("generation-evidence verification requires --result and --receipt pairs"); + } + const keyName = resolveApiKeyName(); + if (keyName === undefined) { + throw new Error(`generation-evidence verification requires ${API_KEY_ENV_NAMES_TEXT}`); + } + const apiKey = process.env[keyName]!; + const samples = await Promise.all(samplePaths.map(async ({ reportPath, receiptPath }) => { + const [reportRaw, receiptRaw] = await Promise.all([ + readFile(reportPath), + readFile(receiptPath, "utf8"), + ]); + return { + report: JSON.parse(reportRaw.toString("utf8")) as unknown, + receipt: JSON.parse(receiptRaw) as unknown, + reportRawSha256: sha256(reportRaw), + }; + })); + const count = await verifyGenerationEvidence(samples, { apiKey }); + console.log(`Verified ${count} distinct OpenRouter generations.`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/bench/src/live.test.ts b/bench/src/live.test.ts index 03cdddd..b5dbbb2 100644 --- a/bench/src/live.test.ts +++ b/bench/src/live.test.ts @@ -10,6 +10,7 @@ import { envelopeOperationalFailure, exactProviderCost, liveCostAccountingComplete, + liveEnv, liveReviewArguments, runLive, resolveLiveTimeoutOverrides, @@ -50,6 +51,30 @@ async function onlyCaseAttempt(runRoot: string): Promise { } describe("live benchmark review mode", () => { + test("opts the managed generation capture proxy into loopback transport", () => { + const env = liveEnv( + "openai/gpt-5.6-luna", + undefined, + "/tmp/screen-profile.json", + "/tmp/home", + "/tmp/runtime", + { + identity: "openrouter:managed-routing", + apiBase: "https://openrouter.ai/api/v1", + apiFormat: "openai-compatible", + }, + { + requestSeconds: null, + totalSeconds: null, + caseProcessMilliseconds: 1_000, + }, + "http://127.0.0.1:4321/api/v1", + ); + + expect(env.POSTIL_QUALIFICATION_CAPTURE_API_BASE).toBe("http://127.0.0.1:4321/api/v1"); + expect(env.POSTIL_ALLOW_PRIVATE_API_BASE).toBe("1"); + }); + test("reports why an authored target was suppressed", () => { const truth = { clean: false, diff --git a/bench/src/live.ts b/bench/src/live.ts index 87526c2..9b79eb2 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -14,7 +14,7 @@ // so no GitHub server, mock or real, is needed. MODEL_API_KEY, LLM_API_KEY, // OPENROUTER_API_KEY, or POSTIL_API_KEY is required and is read from the // caller's environment; it is never logged or printed. -// REVIEW_MODEL or --model is required. +// The shipped default model is used unless REVIEW_MODEL or --model overrides it. // // Scoring uses fixture ground truth: a defect counts as detected when a finding // matches the authored ground-truth region; severity match is tracked @@ -28,11 +28,11 @@ import { execFile as execFileCb } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { join, relative, resolve, sep } from "node:path"; import { promisify } from "node:util"; import { API_KEY_ENV_NAMES_TEXT, forwardApiKey, resolveApiKeyName } from "./api-key"; import { benchmarkCase, type BenchmarkCaseInput, envelopeV1, type Envelope } from "./harness"; +import evaluatorContractSourcePaths from "../evaluator-contract-sources.json"; import { formatCanonicalDecimal, parseCanonicalDecimal, @@ -41,6 +41,7 @@ import { sumCanonicalDecimals, type ProviderContractEvidence, } from "./livemodels-score"; +import { startManagedRequestWindowProxy } from "./request-window"; const execFile = promisify(execFileCb); export const ADMISSION_API_BASE = "https://openrouter.ai:443/api/v1"; @@ -208,6 +209,7 @@ export interface LiveSummary { totalTokens: { prompt: number; completion: number; total: number }; observedProviderCostUsdDecimal: string; costAccountingComplete: boolean; + providerGenerationIds: string[]; errors: number; } @@ -239,66 +241,81 @@ export async function runLive( const timeoutOverrides = resolveLiveTimeoutOverrides(options.timeoutMs); const cases = inputs.map((input) => benchmarkCase.parse(input)); const provider = liveProvider(); - const screeningProfile = options.screenProfilePath === undefined - ? null - : await screeningProfileMetadata(options.screenProfilePath); - await assertBinary(options.binary); - const binarySha256 = createHash("sha256") - .update(await readFile(options.binary)) - .digest("hex"); - const fixtureCorpusSha256 = createHash("sha256") - .update(JSON.stringify(cases)) - .digest("hex"); - const evaluatorSha256 = await evaluatorSourceSha256(); - - const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs"); - const runRoot = await reserveLiveRunRoot(rootDir, options.runId); - await writeLiveRunContract(runRoot, options, timeoutOverrides); - - // Bounded worker pool: a small fixed number of workers pull case indices off a - // shared cursor until the queue drains. Each case writes its result into the - // slot for its original index, so completion order never affects the report. - const results = new Array(cases.length); - const concurrency = Math.max(1, Math.min(options.concurrency ?? DEFAULT_LIVE_CONCURRENCY, cases.length || 1)); - let cursor = 0; - const worker = async (): Promise => { - for (;;) { - const index = cursor++; - if (index >= cases.length) return; - results[index] = await runLiveCaseWithRetry( - cases[index]!, - index, - runRoot, + const providerGenerationIds: string[] = []; + let requestProxy: ReturnType | undefined; + try { + requestProxy = provider.identity === "openrouter:managed-routing" + ? startManagedRequestWindowProxy(provider.apiBase, { + requireGenerationId: true, + onGenerationId: (generationId) => providerGenerationIds.push(generationId), + }) + : undefined; + const screeningProfile = options.screenProfilePath === undefined + ? null + : await screeningProfileMetadata(options.screenProfilePath); + await assertBinary(options.binary); + const binarySha256 = createHash("sha256") + .update(await readFile(options.binary)) + .digest("hex"); + const fixtureCorpusSha256 = createHash("sha256") + .update(JSON.stringify(cases)) + .digest("hex"); + const evaluatorSha256 = await evaluatorSourceSha256(); + + const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs"); + const runRoot = await reserveLiveRunRoot(rootDir, options.runId); + await writeLiveRunContract(runRoot, options, timeoutOverrides); + + // Bounded worker pool: a small fixed number of workers pull case indices off a + // shared cursor until the queue drains. Each case writes its result into the + // slot for its original index, so completion order never affects the report. + const results = new Array(cases.length); + const concurrency = Math.max(1, Math.min(options.concurrency ?? DEFAULT_LIVE_CONCURRENCY, cases.length || 1)); + let cursor = 0; + const worker = async (): Promise => { + for (;;) { + const index = cursor++; + if (index >= cases.length) return; + results[index] = await runLiveCaseWithRetry( + cases[index]!, + index, + runRoot, + options, + timeoutOverrides, + provider, + requestProxy?.apiBase, + ); + } + }; + await Promise.all(Array.from({ length: concurrency }, () => worker())); + + // Results are already index-aligned; sorting by case index is belt-and-braces + // so the written report is deterministically ordered regardless of the pool. + // Strip the internal `stderr` field so the persisted report schema is intact. + const ordered = results + .map((result, index) => ({ result, index })) + .sort((a, b) => a.index - b.index) + .map(({ result }) => { + const { stderr: _stderr, ...rest } = result; + return rest; + }); + return { + summary: summarize( + ordered, options, + binarySha256, + fixtureCorpusSha256, + evaluatorSha256, + provider, + screeningProfile, timeoutOverrides, - ); - } - }; - await Promise.all(Array.from({ length: concurrency }, () => worker())); - - // Results are already index-aligned; sorting by case index is belt-and-braces - // so the written report is deterministically ordered regardless of the pool. - // Strip the internal `stderr` field so the persisted report schema is intact. - const ordered = results - .map((result, index) => ({ result, index })) - .sort((a, b) => a.index - b.index) - .map(({ result }) => { - const { stderr: _stderr, ...rest } = result; - return rest; - }); - return { - summary: summarize( - ordered, - options, - binarySha256, - fixtureCorpusSha256, - evaluatorSha256, - provider, - screeningProfile, - timeoutOverrides, - ), - results: ordered, - }; + providerGenerationIds, + ), + results: ordered, + }; + } finally { + requestProxy?.stop(); + } } const CANONICAL_POSITIVE_SECONDS = /^[1-9][0-9]*$/u; @@ -444,19 +461,23 @@ export async function screeningProfileMetadata(path: string): Promise<{ } export async function evaluatorSourceSha256(): Promise { - const benchRoot = resolve(fileURLToPath(import.meta.url), "..", ".."); - const sources = [ - "fixtures/cases.ts", - "src/api-key.ts", - "src/harness.ts", - "src/live.ts", - "src/livemodels-score.ts", - ]; + const repositoryRoot = resolve(import.meta.dir, "..", ".."); + const sources = evaluatorContractSourcePaths as readonly unknown[]; + if (sources.length === 0 || sources.some((source) => typeof source !== "string" || source.length === 0)) { + throw new Error("evaluator contract source list must contain nonempty repository-relative paths"); + } const hash = createHash("sha256"); for (const source of sources) { - hash.update(`${source}\0`); - hash.update(await readFile(join(benchRoot, source))); - hash.update("\0"); + const path = source as string; + const absolutePath = resolve(repositoryRoot, path); + const relativePath = relative(repositoryRoot, absolutePath); + if (relativePath === "" || relativePath === ".." || relativePath.startsWith(`..${sep}`)) { + throw new Error(`evaluator contract source is outside the repository: ${path}`); + } + hash.update(path, "utf8"); + hash.update(Buffer.from([0])); + hash.update(await readFile(absolutePath)); + hash.update(Buffer.from([0])); } return hash.digest("hex"); } @@ -475,11 +496,22 @@ async function runLiveCaseWithRetry( runRoot: string, options: LiveOptions, timeoutOverrides: LiveTimeoutOverrides, + provider: LiveProvider, + captureApiBase: string | undefined, ): Promise { const maxRetries = options.retries ?? DEFAULT_LIVE_RETRIES; let last: LiveCaseResult | undefined; for (let attempt = 0; attempt <= maxRetries; attempt++) { - last = await runLiveCase(c, index, attempt + 1, runRoot, options, timeoutOverrides); + last = await runLiveCase( + c, + index, + attempt + 1, + runRoot, + options, + timeoutOverrides, + provider, + captureApiBase, + ); // Scored => a valid envelope was produced; that is a normal result (even if // it has findings or false positives), so never retry it. if (last.scored) return last; @@ -584,6 +616,8 @@ async function runLiveCase( runRoot: string, options: LiveOptions, timeoutOverrides: LiveTimeoutOverrides, + provider: LiveProvider, + captureApiBase: string | undefined, ): Promise { const truth = groundTruthOf(c); const base: LiveCaseResult = { @@ -633,8 +667,9 @@ async function runLiveCase( options.screenProfilePath, homeDir, tmpDir, - liveProvider(), + provider, timeoutOverrides, + captureApiBase, ), timeout: timeoutOverrides.caseProcessMilliseconds, maxBuffer: 8 * 1024 * 1024, @@ -769,7 +804,7 @@ export function scorerOperationalFailure( return null; } -interface LiveProvider { +export interface LiveProvider { identity: "openrouter:managed-routing" | "custom"; apiBase: string; apiFormat: string; @@ -784,7 +819,7 @@ function liveProvider(): LiveProvider { }; } -function liveEnv( +export function liveEnv( model: string, scorerModel: string | undefined, screenProfilePath: string | undefined, @@ -792,6 +827,7 @@ function liveEnv( tmpDir: string, provider: LiveProvider, timeoutOverrides: LiveTimeoutOverrides, + captureApiBase?: string, ): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, @@ -821,6 +857,12 @@ function liveEnv( : { POSTIL_BENCH_SCREEN_PROFILE: resolve(screenProfilePath), POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY: "1", + ...(captureApiBase === undefined + ? {} + : { + POSTIL_QUALIFICATION_CAPTURE_API_BASE: captureApiBase, + POSTIL_ALLOW_PRIVATE_API_BASE: "1", + }), }), }; forwardApiKey(env); @@ -955,6 +997,7 @@ function summarize( providerContract: ProviderContractEvidence; } | null, timeoutOverrides: LiveTimeoutOverrides, + providerGenerationIds: string[], ): LiveSummary { const defects = results.filter((r) => r.type === "defect"); const cleans = results.filter((r) => r.type === "clean"); @@ -1034,6 +1077,7 @@ function summarize( }, observedProviderCostUsdDecimal: formatCanonicalDecimal(observedProviderCost), costAccountingComplete: liveCostAccountingComplete(results), + providerGenerationIds, errors: results.filter((r) => r.error !== undefined).length, }; } diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 473c3b5..9f2bca6 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -65,6 +65,7 @@ import { type LiveModelCaseResult, type QualificationPair, } from "./livemodels-score"; +import { evaluatorSourceSha256 } from "./live"; const pair: QualificationPair = { generatorModel: "test/generator", scorerModel: "test/scorer" }; @@ -1362,8 +1363,40 @@ describe("managed admission workflow", () => { const release = await Bun.file( resolve(import.meta.dir, "..", "..", ".github", "workflows", "release.yml"), ).text(); - expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n bench-live:\n/u); - expect(release).toMatch(/bench-live:\n\s+needs: validate-tag\n/u); + const calibration = await Bun.file( + resolve(import.meta.dir, "..", "..", ".github", "workflows", "benchmark-calibration.yml"), + ).text(); + expect(calibration).toContain("name: Reserve the current main calibration source"); + expect(calibration).toContain('refs/tags/postil-calibration-${GITHUB_SHA}'); + expect(calibration).not.toContain("actions/workflows/benchmark-calibration.yml/runs"); + expect(calibration).toContain("max-parallel: 1"); + expect(calibration).toContain("--mode reserve"); + expect(calibration).toContain("name: Attest benchmark sample reservation"); + expect(calibration).toContain("--mode execute"); + expect(calibration).toContain("name: Attest benchmark sample result"); + expect(calibration).toContain("name: Record the attested baseline"); + expect(calibration).toContain("name: Attest the populated baseline"); + expect(calibration).toContain("name: Verify independent calibration generations"); + expect(calibration).toContain("bun run bench:verify-generations --"); + expect(calibration).toContain("--record"); + expect(release).not.toContain("workflow_dispatch"); + expect(release).toContain("name: Require the unique first release run for this tag"); + expect(release).toContain('if [[ "${GITHUB_RUN_ATTEMPT}" != "1" ]]'); + expect(release).toContain('"repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs"'); + expect(release).toContain("This version tag already has another release run."); + expect(release).toContain("group: release-${{ github.ref_name }}"); + expect(release).toContain('gh release view "${GITHUB_REF_NAME}"'); + expect(release).toContain("name: Verify the attested Luna calibration baseline"); + expect(release).toContain("bench/baseline.attestation.json"); + expect(release).toContain('git/ref/tags/postil-calibration-${source_sha}'); + expect(release).toContain( + "--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml", + ); + expect(release).toContain('--signer-digest "$source_sha"'); + expect(release).toContain('--source-digest "$source_sha"'); + expect(release).toContain("--source-ref refs/heads/main"); + expect(release).toMatch(/validate-tag:\n[\s\S]*?permissions:\n\s+contents: read\n\s+actions: read/u); + expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?name: Verify the attested Luna calibration baseline[\s\S]*?\n bench-live-prepare:\n/u); // The gate derives its model from the binary's embedded configuration so // caller drift cannot benchmark a model absent from the release. expect(release).not.toContain("REVIEW_MODEL:"); @@ -1380,40 +1413,114 @@ describe("managed admission workflow", () => { /name: Upload the scorer gate report\n\s+if: always\(\)[\s\S]*if-no-files-found: warn[\s\S]*retention-days: 30/u, ); expect(release.indexOf("bun run scorer-eval --json-out")).toBeLessThan( - release.indexOf("bun run bench:live --"), + release.indexOf("bun run bench:cohort-run --"), ); - expect([...release.matchAll(/bun run bench:live --/gu)]).toHaveLength(3); - for (const sample of [1, 2, 3]) { - expect(release).toContain(`name: Run diff-file live benchmark sample ${sample}`); - expect(release).toContain(`id: live-sample-${sample}`); - expect(release).toContain(`--run-id "release-\${{ github.ref_name }}-sample-${sample}"`); - expect(release).toContain( - `--expected-run-id "release-\${{ github.ref_name }}-sample-${sample}"`, - ); - expect(release).toContain(`--json-out "\${{ runner.temp }}/bench-live-report-${sample}.json"`); - } - expect(release.indexOf("Run diff-file live benchmark sample 1")).toBeLessThan( - release.indexOf("Run diff-file live benchmark sample 2"), + const prepareStart = release.indexOf("\n bench-live-prepare:\n"); + const sampleStart = release.indexOf("\n bench-live-sample:\n"); + const finalStart = release.indexOf("\n bench-live:\n"); + const buildStart = release.indexOf("\n build:\n"); + expect(prepareStart).toBeGreaterThan(-1); + expect(sampleStart).toBeGreaterThan(prepareStart); + expect(finalStart).toBeGreaterThan(sampleStart); + expect(buildStart).toBeGreaterThan(finalStart); + const prepare = release.slice(prepareStart, sampleStart); + const sample = release.slice(sampleStart, finalStart); + const final = release.slice(finalStart, buildStart); + const build = release.slice(buildStart); + + expect(prepare).toMatch(/bench-live-prepare:\n\s+needs: validate-tag\n/u); + expect(prepare).toContain("name: benchmarked-x86_64-unknown-linux-gnu-${{ github.run_attempt }}"); + expect(prepare).toContain("path: target/release/postil"); + expect(prepare).toContain("bun run bench:cohort-create --"); + expect(prepare).toContain("--purpose release"); + expect(prepare).toContain("--out \"${{ runner.temp }}/bench-live-cohort.json\""); + expect(prepare).toContain("uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4"); + expect(prepare).toContain("id-token: write"); + expect(prepare).toContain("attestations: write"); + expect(prepare).toContain("artifact-metadata: write"); + expect(prepare).toContain("bench-live-cohort.attestation.json"); + expect(prepare).toContain("name: bench-live-cohort-${{ github.run_attempt }}"); + + expect(sample).toMatch( + /bench-live-sample:\n\s+needs: \[validate-tag, bench-live-prepare\][\s\S]*?strategy:\n\s+fail-fast: false\n\s+max-parallel: 1\n\s+matrix:\n\s+sample: \[1, 2, 3, 4, 5\]/u, ); - expect(release.indexOf("Run diff-file live benchmark sample 2")).toBeLessThan( - release.indexOf("Run diff-file live benchmark sample 3"), + expect(sample).toContain("name: benchmarked-x86_64-unknown-linux-gnu-${{ github.run_attempt }}"); + expect(sample).toContain("path: target/release"); + expect(sample).toContain("chmod 0755 target/release/postil"); + expect(sample).toContain( + "POSTIL_BIN: ${{ github.workspace }}/target/release/postil", ); - expect(release).toMatch( - /name: Upload the diff-file live reports\n\s+if: always\(\)[\s\S]*bench-live-report-1\.json[\s\S]*bench-live-report-2\.json[\s\S]*bench-live-report-3\.json[\s\S]*retention-days: 30/u, + expect([...sample.matchAll(/bun run bench:cohort-run --/gu)]).toHaveLength(2); + expect(sample).toContain("--mode reserve"); + expect(sample).toContain("--mode execute"); + expect(sample).toContain("name: Attest benchmark sample reservation"); + expect(sample).toContain("reservation.attestation.json"); + expect(sample).toContain("name: Run diff-file live benchmark sample ${{ matrix.sample }}"); + expect(sample).toContain( + '--manifest "${{ runner.temp }}/bench-live-cohort.json"', ); - expect(release).toMatch( - /bun run bench:compare --[\s\S]*--binary "\$\{\{ github\.workspace \}\}\/target\/release\/postil"[\s\S]*--screen-profile \.\.\/provisional-models\.json[\s\S]*--result "\$\{\{ runner\.temp \}\}\/bench-live-report-1\.json"[\s\S]*--result "\$\{\{ runner\.temp \}\}\/bench-live-report-2\.json"[\s\S]*--result "\$\{\{ runner\.temp \}\}\/bench-live-report-3\.json"/u, + expect(sample).toContain("--screen-profile ../provisional-models.json"); + expect(sample).not.toContain("--report-out"); + expect(sample).not.toContain("--receipt-out"); + expect(sample).toContain("uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4"); + expect(sample).toContain("gh attestation verify"); + expect(sample).toContain("--signer-workflow postil-dev/postil-cli/.github/workflows/release.yml"); + expect(sample).toContain("--signer-digest \"${GITHUB_SHA}\""); + expect(sample).toContain("--source-digest \"${GITHUB_SHA}\""); + expect(sample).toContain("--source-ref \"${GITHUB_REF}\""); + expect(sample).toContain("--deny-self-hosted-runners"); + expect(sample).toMatch( + /name: Upload diff-file live benchmark sample \$\{\{ matrix\.sample \}\}\n\s+if: always\(\)[\s\S]*?name: bench-live-sample-\$\{\{ github\.run_attempt \}\}-\$\{\{ matrix\.sample \}\}[\s\S]*?path: \$\{\{ runner\.temp \}\}\/bench-live-sample-\$\{\{ github\.run_attempt \}\}-\$\{\{ matrix\.sample \}\}[\s\S]*?retention-days: 30/u, + ); + + expect(final).toMatch( + /bench-live:\n\s+if: always\(\)\n\s+needs: \[bench-live-prepare, bench-live-sample\]/u, + ); + expect(final).toMatch( + /name: benchmarked-x86_64-unknown-linux-gnu-\$\{\{ github\.run_attempt \}\}\n\s+path: target\/release/u, + ); + expect(final).toContain("pattern: bench-live-sample-${{ github.run_attempt }}-*"); + expect(final).toContain("path: ${{ runner.temp }}/bench-live-reports-download"); + expect(final).toContain("merge-multiple: false"); + expect(final).toContain("name: Verify signed release benchmark evidence"); + expect(final).toContain("name: Verify independent release generations"); + expect(final).toContain("bun run bench:verify-generations --"); + expect(final).toContain("gh attestation verify"); + expect(final).toContain("--deny-self-hosted-runners"); + expect(final).toContain("name: bench-live-cohort-${{ github.run_attempt }}"); + expect(final).toContain('--cohort-manifest "${{ runner.temp }}/bench-live-cohort.json"'); + for (const sample of [1, 2, 3, 4, 5]) { + expect(final).toContain( + `--expected-run-id "release-\${{ github.ref_name }}-\${{ github.run_id }}-\${{ github.run_attempt }}-0${sample}"`, + ); + expect(final).toContain( + `--result "\${{ runner.temp }}/bench-live-reports/slots/0${sample}/report.json"`, + ); + expect(final).toContain( + `--receipt "\${{ runner.temp }}/bench-live-reports/slots/0${sample}/receipt.json"`, + ); + } + expect([...final.matchAll(/--expected-run-id /gu)]).toHaveLength(5); + expect([...final.matchAll(/--result /gu)]).toHaveLength(10); + expect([...final.matchAll(/--receipt /gu)]).toHaveLength(10); + expect(final).toMatch( + /bun run bench:compare --[\s\S]*--binary "\$\{\{ github\.workspace \}\}\/target\/release\/postil"[\s\S]*--screen-profile \.\.\/provisional-models\.json/u, ); - expect([...release.matchAll(/continue-on-error: true/gu)]).toHaveLength(3); - expect(release).toContain("SAMPLE_1_OUTCOME: ${{ steps.live-sample-1.outcome }}"); - expect(release).toContain("SAMPLE_2_OUTCOME: ${{ steps.live-sample-2.outcome }}"); - expect(release).toContain("SAMPLE_3_OUTCOME: ${{ steps.live-sample-3.outcome }}"); + expect(final).toContain("SAMPLE_JOB_RESULT: ${{ needs.bench-live-sample.result }}"); + expect(final).toContain('if [[ "${SAMPLE_JOB_RESULT}" != "success" ]]'); + expect(final).toContain("At least one live benchmark sample failed before comparison."); + expect(release).toContain("OPENROUTER_API_KEY has insufficient credit for the release benchmark reserve."); + expect(release).toContain('echo "OpenRouter credential accepted."'); + expect(release).not.toContain("remaining=\"$(jq -r '.data.limit_remaining"); + expect(release).not.toMatch(/credential accepted.*remaining/iu); expect(release).not.toContain("bench-live-report-1.json.partial"); expect(release).not.toContain("bench-live-report-2.json.partial"); expect(release).not.toContain("bench-live-report-3.json.partial"); + expect(release).not.toContain("bench-live-report-4.json.partial"); + expect(release).not.toContain("bench-live-report-5.json.partial"); expect(release).not.toContain("bench_live_override_reason"); expect(release).not.toContain("OVERRIDE_REASON"); - expect(release).toMatch(/build:\n\s+needs: \[validate-tag, bench-live\]/u); + expect(build).toMatch(/build:\n\s+needs: \[validate-tag, bench-live\]/u); let checkedReferences = 0; const workflowGlob = new Bun.Glob("*.yml"); for await (const workflowName of workflowGlob.scan(resolve(import.meta.dir, "..", "..", ".github", "workflows"))) { @@ -1471,6 +1578,39 @@ describe("qualification report", () => { expect(EVALUATOR_CONTRACT_SOURCE_PATHS).toContain("bench/package.json"); expect(EVALUATOR_CONTRACT_SOURCE_PATHS).toContain("bench/bun.lock"); }); + + test("covers every evaluator authority source and changes when a listed source changes", async () => { + const required = [ + ".github/workflows/benchmark-calibration.yml", + ".github/workflows/release.yml", + "bench/evaluator-contract-sources.json", + "bench/src/cohort.ts", + "bench/src/cohort-run.ts", + "bench/src/compare-baseline.ts", + "bench/src/live.ts", + "bench/src/run.ts", + ]; + for (const source of required) expect(EVALUATOR_CONTRACT_SOURCE_PATHS).toContain(source); + expect(new Set(EVALUATOR_CONTRACT_SOURCE_PATHS).size).toBe(EVALUATOR_CONTRACT_SOURCE_PATHS.length); + const repositoryRoot = resolve(import.meta.dir, "../.."); + for (const source of EVALUATOR_CONTRACT_SOURCE_PATHS) { + expect((await readFile(resolve(repositoryRoot, source))).byteLength).toBeGreaterThan(0); + } + expect(await evaluatorSourceSha256()).toMatch(/^[0-9a-f]{64}$/u); + + const root = await mkdtemp(resolve(tmpdir(), "postil-evaluator-source-")); + const source = resolve(root, "source.ts"); + try { + const listedPath = "bench/src/live.ts"; + await writeFile(source, "authority source one"); + const before = hashNamedSources([[listedPath, await readFile(source)]]); + await writeFile(source, "authority source two"); + const after = hashNamedSources([[listedPath, await readFile(source)]]); + expect(after).not.toBe(before); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); test("matches the runtime named-source framing vector", () => { expect(hashNamedSources([ ["a.txt", Buffer.from("alpha")], diff --git a/bench/src/request-window.test.ts b/bench/src/request-window.test.ts index dcba6c5..205e5d0 100644 --- a/bench/src/request-window.test.ts +++ b/bench/src/request-window.test.ts @@ -143,4 +143,39 @@ describe("managed request-window governor", () => { expect(accepted.status).toBe(200); expect(starts[1]! - starts[0]!).toBeGreaterThanOrEqual(40); }); + + test("captures required OpenRouter generation identities and rejects their absence", async () => { + let includeIdentity = true; + const upstream = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch() { + return Response.json({ accepted: true }, { + headers: includeIdentity ? { "X-Generation-Id": "gen-fixture-1" } : {}, + }); + }, + }); + servers.push(upstream); + const generationIds: string[] = []; + const proxy = startManagedRequestWindowProxy(`${new URL(upstream.url).origin}/api/v1`, { + requireGenerationId: true, + onGenerationId: (generationId) => generationIds.push(generationId), + }); + proxies.push(proxy); + const send = () => fetch(`${proxy.apiBase}/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + + const accepted = await send(); + expect(accepted.status).toBe(200); + expect(accepted.headers.get("x-generation-id")).toBe("gen-fixture-1"); + expect(generationIds).toEqual(["gen-fixture-1"]); + + includeIdentity = false; + const rejected = await send(); + expect(rejected.status).toBe(502); + expect(generationIds).toEqual(["gen-fixture-1"]); + }); }); diff --git a/bench/src/request-window.ts b/bench/src/request-window.ts index a33aabc..173be85 100644 --- a/bench/src/request-window.ts +++ b/bench/src/request-window.ts @@ -12,6 +12,8 @@ export interface RequestWindowOptions { retryAfterCapMs?: number; now?: Clock; sleep?: Sleep; + requireGenerationId?: boolean; + onGenerationId?: (generationId: string) => void; } /** @@ -161,6 +163,18 @@ export function startManagedRequestWindowProxy( return Response.json({ error: "managed provider request failed" }, { status: 502 }); } await governor.observeRetryAfter(response.headers.get("retry-after")); + if (response.ok) { + const generationId = response.headers.get("x-generation-id")?.trim(); + if (options.requireGenerationId === true && + (generationId === undefined || !/^gen-[A-Za-z0-9_-]+$/u.test(generationId))) { + return Response.json({ error: "managed provider response omitted its generation identity" }, { + status: 502, + }); + } + if (generationId !== undefined && /^gen-[A-Za-z0-9_-]+$/u.test(generationId)) { + options.onGenerationId?.(generationId); + } + } return new Response(response.body, { status: response.status, headers: selectedHeaders(response.headers, [ @@ -168,6 +182,7 @@ export function startManagedRequestWindowProxy( "retry-after", "x-request-id", "x-openrouter-request-id", + "x-generation-id", ]), }); }, @@ -183,8 +198,9 @@ export function startManagedRequestWindowProxy( export async function withManagedRequestWindowProxy( upstreamApiBase: string, work: (proxy: ManagedRequestWindowProxy) => Promise, + options: RequestWindowOptions & { fetchImpl?: typeof fetch } = {}, ): Promise { - const proxy = startManagedRequestWindowProxy(upstreamApiBase); + const proxy = startManagedRequestWindowProxy(upstreamApiBase, options); try { return await work(proxy); } finally { diff --git a/src/alerts.rs b/src/alerts.rs new file mode 100644 index 0000000..356fcd7 --- /dev/null +++ b/src/alerts.rs @@ -0,0 +1,798 @@ +//! Operator-only event-driven iLert alert follower. + +use std::io::{self, Write}; +use std::path::Path; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use reqwest::header::{ACCEPT, AUTHORIZATION, HeaderValue, RETRY_AFTER}; +use serde::Deserialize; +use tokio::time::{Instant, sleep, timeout, timeout_at}; + +use crate::{credentials, llm::secure_http_client_async, login}; + +const ALERT_STREAM_PATH: &str = "/api/operator/alerts/stream"; +const ALERT_STREAM_MAX_PENDING_BYTES: usize = 512 * 1024; +const ALERT_STREAM_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +const ALERT_STREAM_PROBE_TIMEOUT: Duration = Duration::from_secs(75); +const ALERT_STREAM_SILENCE_TIMEOUT: Duration = Duration::from_secs(45); +const ALERT_STREAM_RECONNECT_BASE: Duration = Duration::from_secs(3); +const ALERT_STREAM_RECONNECT_MAX: Duration = Duration::from_secs(60); +const ALERT_STREAM_RETRY_AFTER_MAX: Duration = Duration::from_secs(60 * 60); +const ALERT_STREAM_DEGRADED_AFTER_FAILURES: u32 = 3; +const POSTGRES_MAX_SEQUENCE: u64 = i64::MAX as u64; + +#[derive(Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct OperatorAlert { + sequence: String, + alert_id: String, + event_type: String, + status: String, + priority: String, + summary: String, +} + +pub async fn run_watch(once: bool, probe: bool) -> Result { + let credentials_path = credentials::default_path()?; + let _watch_lock = credentials::AlertWatchLock::acquire(&credentials_path)?; + let interrupted_exit = if once || probe { 130 } else { 0 }; + if probe { + tokio::select! { + result = timeout(ALERT_STREAM_PROBE_TIMEOUT, watch(&credentials_path, once, probe)) => { + result.context("operator alert notification probe timed out")? + }, + signal = tokio::signal::ctrl_c() => { + signal.context("waiting for alert watcher shutdown")?; + Ok(interrupted_exit) + } + } + } else { + tokio::select! { + result = watch(&credentials_path, once, probe) => result, + signal = tokio::signal::ctrl_c() => { + signal.context("waiting for alert watcher shutdown")?; + Ok(interrupted_exit) + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CursorState { + issuer: String, + sequence: u64, +} + +fn cursor_state(issuer: &str, sequence: Option) -> Option { + sequence.map(|sequence| CursorState { + issuer: issuer.to_string(), + sequence, + }) +} + +async fn watch(credentials_path: &Path, once: bool, probe: bool) -> Result { + let mut cursor: Option = None; + let mut consecutive_failures = 0_u32; + let mut degraded = false; + loop { + let outcome = + watch_connection(credentials_path, cursor.as_ref(), once, probe, degraded).await?; + if once && outcome.notification_received { + return Ok(0); + } + if probe && outcome.probe_validated { + return Ok(0); + } + + if update_watch_health(&mut consecutive_failures, &mut degraded, &outcome) { + eprintln!("postil: operator alert notifications unavailable; reconnecting"); + } + let reconnect_delay = reconnect_delay(consecutive_failures, outcome.retry_after); + cursor = outcome.last_cursor; + sleep(reconnect_delay).await; + } +} + +struct ConnectionOutcome { + last_cursor: Option, + notification_received: bool, + probe_validated: bool, + stable_connection: bool, + healthy_reconnect: bool, + retry_after: Option, +} + +impl ConnectionOutcome { + fn retry( + last_cursor: Option, + retry_after: Option, + healthy_reconnect: bool, + ) -> Self { + Self { + last_cursor, + notification_received: false, + probe_validated: false, + stable_connection: false, + healthy_reconnect, + retry_after, + } + } +} + +fn update_watch_health( + consecutive_failures: &mut u32, + degraded: &mut bool, + outcome: &ConnectionOutcome, +) -> bool { + if outcome.stable_connection && *degraded { + *degraded = false; + } + if outcome.healthy_reconnect { + *consecutive_failures = 0; + } else if outcome.stable_connection { + *consecutive_failures = 1; + } else { + *consecutive_failures = consecutive_failures.saturating_add(1); + } + let became_degraded = + !*degraded && *consecutive_failures >= ALERT_STREAM_DEGRADED_AFTER_FAILURES; + if became_degraded { + *degraded = true; + } + became_degraded +} + +fn mark_connection_stable(stable_connection: &mut bool, frame_stable: bool) -> bool { + if frame_stable && !*stable_connection { + *stable_connection = true; + true + } else { + false + } +} + +async fn watch_connection( + credentials_path: &Path, + last_cursor: Option<&CursorState>, + once: bool, + probe: bool, + announce_recovery: bool, +) -> Result { + let session = match timeout( + ALERT_STREAM_CONNECT_TIMEOUT, + login::resolve_stored_alert_session(credentials_path), + ) + .await + { + Ok(Ok(Some(session))) => session, + Ok(Ok(None)) => anyhow::bail!("postil login required for operator alert notifications"), + Ok(Err(error)) => match login::token_resolution_retry_delay(&error) { + Some(delay) => { + return Ok(ConnectionOutcome::retry( + last_cursor.cloned(), + (!delay.is_zero()).then_some(delay), + false, + )); + } + None => return Err(error), + }, + Err(_) => { + return Ok(ConnectionOutcome::retry(last_cursor.cloned(), None, false)); + } + }; + let issuer = session.issuer; + let token = session.token; + let mut cursor = match last_cursor.filter(|cursor| cursor.issuer == issuer) { + Some(cursor) => Some(cursor.sequence), + None => credentials::read_alert_cursor(credentials_path, &issuer)?, + }; + let endpoint = format!("{}{ALERT_STREAM_PATH}", issuer.trim_end_matches('/')); + let client = match timeout( + ALERT_STREAM_CONNECT_TIMEOUT, + secure_http_client_async(&issuer), + ) + .await + { + Ok(Ok(client)) => client, + Ok(Err(_)) | Err(_) => { + return Ok(ConnectionOutcome::retry( + cursor_state(&issuer, cursor), + None, + false, + )); + } + }; + let mut authorization = HeaderValue::from_str(&format!("Bearer {token}")) + .context("stored postil login contains an invalid access credential")?; + authorization.set_sensitive(true); + let mut request = client + .get(&endpoint) + .header(ACCEPT, "text/event-stream") + .header(AUTHORIZATION, authorization); + if let Some(last_event_id) = cursor { + request = request.header("last-event-id", last_event_id.to_string()); + } + let mut response = match timeout(ALERT_STREAM_CONNECT_TIMEOUT, request.send()).await { + Ok(Ok(response)) => response, + Ok(Err(_)) | Err(_) => { + return Ok(ConnectionOutcome::retry( + cursor_state(&issuer, cursor), + None, + false, + )); + } + }; + let requested_delay = retry_after_delay(&response); + match response.status().as_u16() { + 200 => {} + 401 => { + let current = timeout( + ALERT_STREAM_CONNECT_TIMEOUT, + login::resolve_stored_alert_session(credentials_path), + ) + .await; + match current { + Ok(Ok(Some(current))) if current.issuer != issuer || current.token != token => { + return Ok(ConnectionOutcome::retry( + cursor_state(&issuer, cursor), + Some(Duration::ZERO), + true, + )); + } + Ok(Err(error)) if login::token_resolution_retry_delay(&error).is_some() => { + return Ok(ConnectionOutcome::retry( + cursor_state(&issuer, cursor), + login::token_resolution_retry_delay(&error) + .filter(|delay| !delay.is_zero()), + false, + )); + } + Err(_) => { + return Ok(ConnectionOutcome::retry( + cursor_state(&issuer, cursor), + None, + false, + )); + } + _ => anyhow::bail!( + "operator alert authorization was rejected; run `postil login` again" + ), + } + } + 404 => anyhow::bail!("operator alert notifications are unavailable for this login"), + 408 | 425 | 429 | 500 | 502 | 503 | 504 => { + return Ok(ConnectionOutcome::retry( + cursor_state(&issuer, cursor), + requested_delay, + false, + )); + } + status => anyhow::bail!("operator alert notification stream returned HTTP {status}"), + } + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + anyhow::ensure!( + content_type + .split(';') + .next() + .is_some_and(|value| value.trim().eq_ignore_ascii_case("text/event-stream")), + "operator alert notification stream returned an invalid content type" + ); + + let mut decoder = SseDecoder::default(); + let mut frame_deadline = Instant::now() + ALERT_STREAM_SILENCE_TIMEOUT; + let mut stable_connection = false; + let mut routine_close = false; + let mut stream_retry_after = None; + loop { + let chunk = match timeout_at(frame_deadline, response.chunk()).await { + Ok(Ok(Some(chunk))) => chunk, + Ok(Ok(None)) => { + return Ok(ConnectionOutcome { + last_cursor: cursor_state(&issuer, cursor), + notification_received: false, + probe_validated: false, + stable_connection, + healthy_reconnect: routine_close, + retry_after: stream_retry_after, + }); + } + Ok(Err(_)) | Err(_) => { + return Ok(ConnectionOutcome { + last_cursor: cursor_state(&issuer, cursor), + notification_received: false, + probe_validated: false, + stable_connection, + healthy_reconnect: false, + retry_after: stream_retry_after, + }); + } + }; + let frames = decoder.push(&chunk)?; + if !frames.is_empty() { + frame_deadline = Instant::now() + ALERT_STREAM_SILENCE_TIMEOUT; + } + for frame in frames { + let (event, requested_retry, frame_stable, frame_routine_close) = match frame { + SseFrame::Control { + retry_after, + stable, + routine_reconnect, + } => (None, retry_after, stable, routine_reconnect), + SseFrame::Event { event, retry_after } => (Some(event), retry_after, true, false), + }; + if requested_retry.is_some() { + stream_retry_after = requested_retry; + } + routine_close |= frame_routine_close; + if mark_connection_stable(&mut stable_connection, frame_stable) && announce_recovery { + eprintln!("postil: operator alert notifications recovered"); + } + if probe { + return Ok(ConnectionOutcome { + last_cursor: cursor_state(&issuer, cursor), + notification_received: false, + probe_validated: true, + stable_connection, + healthy_reconnect: true, + retry_after: stream_retry_after, + }); + } + let Some(event) = event else { + continue; + }; + let alert: OperatorAlert = serde_json::from_str(&event.data) + .context("operator alert notification contained invalid JSON")?; + let alert_sequence = parse_sequence(&alert.sequence)?; + anyhow::ensure!( + alert_sequence == event.id, + "operator alert notification sequence did not match its SSE cursor" + ); + if let Some(previous) = cursor { + anyhow::ensure!( + event.id > previous, + "operator alert notification sequence was not increasing" + ); + } + let delivery = credentials::deliver_alert_with_cursor( + credentials_path, + &issuer, + &token, + event.id, + || { + let mut output = io::stdout().lock(); + writeln!( + output, + "iLert {} alert {} {}: {} ({})", + terminal_text(&alert.priority), + terminal_text(&alert.alert_id), + terminal_text(&alert.event_type), + terminal_text(&alert.summary), + terminal_text(&alert.status) + )?; + output.flush()?; + Ok(()) + }, + ) + .await?; + match delivery { + credentials::AlertCursorDelivery::Delivered(sequence) => { + cursor = Some(sequence); + if once { + return Ok(ConnectionOutcome { + last_cursor: cursor_state(&issuer, cursor), + notification_received: true, + probe_validated: false, + stable_connection: true, + healthy_reconnect: true, + retry_after: stream_retry_after, + }); + } + } + credentials::AlertCursorDelivery::AlreadyRecorded(sequence) => { + cursor = Some(sequence); + } + credentials::AlertCursorDelivery::SessionChanged => { + return Ok(ConnectionOutcome { + last_cursor: cursor_state(&issuer, cursor), + notification_received: false, + probe_validated: false, + stable_connection: true, + healthy_reconnect: true, + retry_after: Some(Duration::ZERO), + }); + } + } + } + } +} + +fn retry_after_delay(response: &reqwest::Response) -> Option { + response + .headers() + .get(RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())) + .and_then(|value| value.parse::().ok()) + .map(Duration::from_secs) + .map(|delay| delay.min(ALERT_STREAM_RETRY_AFTER_MAX)) +} + +fn reconnect_delay(failures: u32, requested: Option) -> Duration { + if let Some(requested) = requested { + return requested.min(ALERT_STREAM_RETRY_AFTER_MAX); + } + let exponent = failures.saturating_sub(1).min(4); + let multiplier = 1_u32 << exponent; + let base = ALERT_STREAM_RECONNECT_BASE + .saturating_mul(multiplier) + .min(ALERT_STREAM_RECONNECT_MAX); + let jitter_millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .subsec_millis() as u64; + base.saturating_add(Duration::from_millis(jitter_millis)) + .min(ALERT_STREAM_RECONNECT_MAX) +} + +fn parse_sequence(value: &str) -> Result { + anyhow::ensure!( + !value.is_empty() + && value.len() <= 19 + && value.bytes().all(|byte| byte.is_ascii_digit()) + && (value == "0" || !value.starts_with('0')), + "operator alert notification contained an invalid sequence" + ); + let sequence = value + .parse::() + .context("operator alert notification sequence was out of range")?; + anyhow::ensure!( + sequence <= POSTGRES_MAX_SEQUENCE, + "operator alert notification sequence was out of range" + ); + Ok(sequence) +} + +fn terminal_text(value: &str) -> String { + value + .chars() + .map(|character| { + if unsafe_terminal_character(character) { + ' ' + } else { + character + } + }) + .take(512) + .collect() +} + +fn unsafe_terminal_character(character: char) -> bool { + character.is_control() + || matches!( + character, + '\u{061c}' + | '\u{200b}'..='\u{200f}' + | '\u{202a}'..='\u{202e}' + | '\u{2060}'..='\u{206f}' + | '\u{feff}' + ) +} + +#[derive(Debug, PartialEq, Eq)] +enum SseFrame { + Control { + retry_after: Option, + stable: bool, + routine_reconnect: bool, + }, + Event { + event: SseEvent, + retry_after: Option, + }, +} + +#[derive(Debug, PartialEq, Eq)] +struct SseEvent { + id: u64, + data: String, +} + +#[derive(Default)] +struct SseDecoder { + pending: Vec, +} + +impl SseDecoder { + fn push(&mut self, bytes: &[u8]) -> Result> { + self.pending.extend_from_slice(bytes); + anyhow::ensure!( + self.pending.len() <= ALERT_STREAM_MAX_PENDING_BYTES, + "operator alert notification exceeded the stream buffer limit" + ); + let mut frames = Vec::new(); + while let Some((end, consumed)) = event_boundary(&self.pending) { + let block = self.pending.drain(..consumed).collect::>(); + let mut id = None; + let mut data = Vec::new(); + let mut control = false; + let mut retry_after = None; + let mut stable = false; + let mut routine_reconnect = false; + for line in sse_lines(&block[..end])? { + if let Some(comment) = line.strip_prefix(':') { + control = true; + match comment.trim_start() { + "keepalive" => stable = true, + "replay batch" | "rotate" => { + stable = true; + routine_reconnect = true; + } + _ => {} + } + continue; + } + if let Some(value) = line.strip_prefix("retry:") { + control = true; + let value = value.strip_prefix(' ').unwrap_or(value); + retry_after = value + .bytes() + .all(|byte| byte.is_ascii_digit()) + .then(|| value.parse::().ok()) + .flatten() + .map(Duration::from_millis) + .map(|delay| delay.min(ALERT_STREAM_RETRY_AFTER_MAX)); + continue; + } + if let Some(value) = line.strip_prefix("id:") { + id = Some(parse_sequence(value.strip_prefix(' ').unwrap_or(value))?); + } else if let Some(value) = line.strip_prefix("data:") { + data.push(value.strip_prefix(' ').unwrap_or(value)); + } + } + if data.is_empty() { + if control { + frames.push(SseFrame::Control { + retry_after, + stable, + routine_reconnect, + }); + } + continue; + } + frames.push(SseFrame::Event { + event: SseEvent { + id: id.ok_or_else(|| { + anyhow::anyhow!("operator alert notification omitted its SSE cursor") + })?, + data: data.join("\n"), + }, + retry_after, + }); + } + Ok(frames) + } +} + +fn sse_lines(bytes: &[u8]) -> Result> { + let mut lines = Vec::new(); + let mut start = 0; + while start < bytes.len() { + let end = bytes[start..] + .iter() + .position(|byte| matches!(byte, b'\r' | b'\n')) + .map(|offset| start + offset) + .unwrap_or(bytes.len()); + lines.push( + std::str::from_utf8(&bytes[start..end]) + .context("operator alert notification was not UTF-8")?, + ); + if end == bytes.len() { + break; + } + start = end + 1; + if bytes[end] == b'\r' && bytes.get(start) == Some(&b'\n') { + start += 1; + } + } + Ok(lines) +} + +fn event_boundary(bytes: &[u8]) -> Option<(usize, usize)> { + let mut start = 0; + loop { + let end = bytes[start..] + .iter() + .position(|byte| matches!(byte, b'\r' | b'\n')) + .map(|offset| start + offset)?; + let mut next = end + 1; + if bytes[end] == b'\r' && bytes.get(next) == Some(&b'\n') { + next += 1; + } + if end == start { + return Some((start, next)); + } + start = next; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_fragmented_events_and_exposes_keepalives() { + let mut decoder = SseDecoder::default(); + assert!(decoder.push(b"retry: 3000\n: con").unwrap().is_empty()); + assert_eq!( + decoder + .push(b"nected\n\nid: 12\nevent: alert-created\ndata: {\"sequence\":\"12\"}\n\n") + .unwrap(), + vec![ + SseFrame::Control { + retry_after: Some(Duration::from_secs(3)), + stable: false, + routine_reconnect: false, + }, + SseFrame::Event { + event: SseEvent { + id: 12, + data: "{\"sequence\":\"12\"}".into(), + }, + retry_after: None, + }, + ] + ); + } + + #[test] + fn distinguishes_service_rotation_and_replay_from_initial_connection() { + let mut decoder = SseDecoder::default(); + assert_eq!( + decoder + .push(b": keepalive\n\n: rotate\n\nretry: 100\n: replay batch\n\n") + .unwrap(), + vec![ + SseFrame::Control { + retry_after: None, + stable: true, + routine_reconnect: false, + }, + SseFrame::Control { + retry_after: None, + stable: true, + routine_reconnect: true, + }, + SseFrame::Control { + retry_after: Some(Duration::from_millis(100)), + stable: true, + routine_reconnect: true, + }, + ] + ); + } + + #[test] + fn degraded_health_latches_until_one_stable_connection() { + let unstable = ConnectionOutcome::retry(None, None, false); + let mut consecutive_failures = 0; + let mut degraded = false; + + assert!(!update_watch_health( + &mut consecutive_failures, + &mut degraded, + &unstable + )); + assert!(!update_watch_health( + &mut consecutive_failures, + &mut degraded, + &unstable + )); + assert!(update_watch_health( + &mut consecutive_failures, + &mut degraded, + &unstable + )); + assert!(degraded); + assert!(!update_watch_health( + &mut consecutive_failures, + &mut degraded, + &unstable + )); + + let stable = ConnectionOutcome { + last_cursor: None, + notification_received: false, + probe_validated: false, + stable_connection: true, + healthy_reconnect: false, + retry_after: None, + }; + assert!(!update_watch_health( + &mut consecutive_failures, + &mut degraded, + &stable + )); + assert!(!degraded); + assert_eq!(consecutive_failures, 1); + + let mut connection_stable = false; + assert!(!mark_connection_stable(&mut connection_stable, false)); + assert!(mark_connection_stable(&mut connection_stable, true)); + assert!(!mark_connection_stable(&mut connection_stable, true)); + } + + #[test] + fn decodes_crlf_cr_only_and_mixed_line_endings() { + for bytes in [ + b"id: 13\r\ndata: {\"sequence\":\"13\"}\r\n\r\n".as_slice(), + b"id: 13\rdata: {\"sequence\":\"13\"}\r\r".as_slice(), + b"id: 13\ndata: {\"sequence\":\"13\"}\r\n\n".as_slice(), + ] { + let mut decoder = SseDecoder::default(); + assert_eq!( + decoder.push(bytes).unwrap(), + vec![SseFrame::Event { + event: SseEvent { + id: 13, + data: "{\"sequence\":\"13\"}".into(), + }, + retry_after: None, + }] + ); + } + } + + #[test] + fn rejects_unbounded_cursorless_or_noncanonical_data() { + let mut decoder = SseDecoder::default(); + assert!(decoder.push(b"data: {}\n\n").is_err()); + let mut decoder = SseDecoder::default(); + assert!(decoder.push(b"id: 01\ndata: {}\n\n").is_err()); + let mut decoder = SseDecoder::default(); + assert!( + decoder + .push(&vec![b'x'; ALERT_STREAM_MAX_PENDING_BYTES + 1]) + .is_err() + ); + } + + #[test] + fn parses_the_service_notification_contract() { + let alert: OperatorAlert = serde_json::from_str( + r#"{"sequence":"42","alertId":"1533","eventType":"alert-created","status":"PENDING","priority":"HIGH","summary":"Review failed"}"#, + ) + .unwrap(); + assert_eq!(parse_sequence(&alert.sequence).unwrap(), 42); + assert_eq!(alert.alert_id, "1533"); + } + + #[test] + fn strips_terminal_controls_and_directional_formatting() { + assert_eq!( + terminal_text("HIGH\u{1b}[31m\n\u{202e}alert\u{2066}"), + "HIGH [31m alert " + ); + } + + #[test] + fn backoff_honors_bounded_retry_after() { + assert_eq!( + reconnect_delay(1, Some(Duration::from_millis(100))), + Duration::from_millis(100) + ); + assert_eq!( + reconnect_delay(1, Some(Duration::from_secs(120))), + Duration::from_secs(120) + ); + assert_eq!( + reconnect_delay(10, Some(Duration::from_secs(7200))), + ALERT_STREAM_RETRY_AFTER_MAX + ); + assert!(reconnect_delay(1, None) >= ALERT_STREAM_RECONNECT_BASE); + } +} diff --git a/src/cli.rs b/src/cli.rs index b4019d5..0f9f638 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -56,6 +56,16 @@ pub enum ForgeArg { #[derive(Subcommand)] #[allow(clippy::large_enum_variant)] // Review carries the full flag set by design. pub enum Command { + /// Follow operator iLert alerts through the authenticated event stream. + #[command(hide = true)] + AlertWatch { + /// Exit after the first delivered notification. + #[arg(long, hide = true, conflicts_with = "probe")] + once: bool, + /// Exit after an authenticated stream connection is established. + #[arg(long, hide = true, conflicts_with = "once")] + probe: bool, + }, /// Probe machine-readable CLI capabilities without external access. Capabilities { /// Require and print an exact publication-plan contract identifier. @@ -692,4 +702,38 @@ mod tests { assert!(matches!(parsed.command, Command::Logout)); assert!(Cli::try_parse_from(["postil", "logout", "--org", "runatlas-is"]).is_err()); } + + #[test] + fn operator_alert_watch_is_hidden_but_parseable() { + let parsed = Cli::try_parse_from(["postil", "alert-watch"]).unwrap(); + assert!(matches!( + parsed.command, + Command::AlertWatch { + once: false, + probe: false + } + )); + let parsed = Cli::try_parse_from(["postil", "alert-watch", "--once"]).unwrap(); + assert!(matches!( + parsed.command, + Command::AlertWatch { + once: true, + probe: false + } + )); + let parsed = Cli::try_parse_from(["postil", "alert-watch", "--probe"]).unwrap(); + assert!(matches!( + parsed.command, + Command::AlertWatch { + once: false, + probe: true + } + )); + assert!(Cli::try_parse_from(["postil", "alert-watch", "--once", "--probe"]).is_err()); + let help = Cli::try_parse_from(["postil", "--help"]) + .err() + .expect("help should exit without parsing a command") + .to_string(); + assert!(!help.contains("alert-watch")); + } } diff --git a/src/config.rs b/src/config.rs index 91b67cd..a040d00 100644 --- a/src/config.rs +++ b/src/config.rs @@ -44,6 +44,14 @@ const BENCH_BUN_LOCK: &str = include_str!("../bench/bun.lock"); const EVALUATOR_CONTRACT_PATHS_JSON: &str = include_str!("../bench/evaluator-contract-sources.json"); const EVALUATOR_CONTRACT_SOURCES: &[(&str, &str)] = &[ + ( + ".github/workflows/benchmark-calibration.yml", + include_str!("../.github/workflows/benchmark-calibration.yml"), + ), + ( + ".github/workflows/release.yml", + include_str!("../.github/workflows/release.yml"), + ), ( "bench/admission-manifest-candidate-vector.json", include_str!("../bench/admission-manifest-candidate-vector.json"), @@ -67,10 +75,27 @@ const EVALUATOR_CONTRACT_SOURCES: &[(&str, &str)] = &[ "bench/src/attribution.ts", include_str!("../bench/src/attribution.ts"), ), + ( + "bench/src/cohort.ts", + include_str!("../bench/src/cohort.ts"), + ), + ( + "bench/src/cohort-run.ts", + include_str!("../bench/src/cohort-run.ts"), + ), + ( + "bench/src/compare-baseline.ts", + include_str!("../bench/src/compare-baseline.ts"), + ), + ( + "bench/src/generation-evidence.ts", + include_str!("../bench/src/generation-evidence.ts"), + ), ( "bench/src/harness.ts", include_str!("../bench/src/harness.ts"), ), + ("bench/src/live.ts", include_str!("../bench/src/live.ts")), ( "bench/src/livemodels-score.ts", include_str!("../bench/src/livemodels-score.ts"), diff --git a/src/credentials.rs b/src/credentials.rs index 4c26abf..6b57430 100644 --- a/src/credentials.rs +++ b/src/credentials.rs @@ -23,6 +23,8 @@ pub const CREDENTIALS_VERSION: u32 = 3; pub const LEGACY_CREDENTIALS_VERSION: u32 = 1; pub const LEGACY_REFRESH_CREDENTIALS_VERSION: u32 = 2; const PENDING_REVOCATIONS_VERSION: u32 = 1; +const ALERT_CURSOR_VERSION: u32 = 1; +const POSTGRES_MAX_SEQUENCE: u64 = i64::MAX as u64; // The refresh exchange has separately bounded send and body-read phases. A // second local process waits long enough for both before asking the caller to // retry, so concurrent agent runs converge on one token rotation. @@ -79,6 +81,14 @@ struct PendingRevocations { revocations: Vec, } +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct AlertCursor { + version: u32, + issuer: String, + sequence: u64, +} + impl Credentials { /// Fails closed: an `expiresAt` that will not parse is treated as /// already expired rather than trusted, so a corrupted file never grants @@ -221,6 +231,85 @@ pub fn write_pending(credentials_path: &Path, revocations: &[PendingRevocation]) ) } +pub fn read_alert_cursor(credentials_path: &Path, issuer: &str) -> Result> { + let path = alert_cursor_path(credentials_path)?; + let raw = match fs::read_to_string(&path) { + Ok(raw) => raw, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), + }; + let cursor: AlertCursor = + serde_json::from_str(&raw).with_context(|| format!("parsing {}", path.display()))?; + anyhow::ensure!( + cursor.version == ALERT_CURSOR_VERSION, + "unsupported operator alert cursor version {}", + cursor.version + ); + anyhow::ensure!( + cursor.sequence <= POSTGRES_MAX_SEQUENCE, + "operator alert cursor sequence is out of range" + ); + if cursor.issuer != issuer { + return Ok(None); + } + Ok(Some(cursor.sequence)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AlertCursorDelivery { + Delivered(u64), + AlreadyRecorded(u64), + SessionChanged, +} + +/// Delivers one alert and advances its cursor while holding the stored-login +/// lock. Delivery is intentionally at-least-once: a crash after the callback +/// flushes but before the cursor rename can replay the alert, while advancing +/// first could silently lose it. +pub async fn deliver_alert_with_cursor( + credentials_path: &Path, + issuer: &str, + expected_token: &str, + sequence: u64, + deliver: F, +) -> Result +where + F: FnOnce() -> Result<()>, +{ + anyhow::ensure!( + sequence <= POSTGRES_MAX_SEQUENCE, + "operator alert cursor sequence is out of range" + ); + let _lock = CredentialLock::acquire(credentials_path).await?; + let Some(active) = read(credentials_path)? else { + return Ok(AlertCursorDelivery::SessionChanged); + }; + if active.version != CREDENTIALS_VERSION + || active.issuer.as_deref() != Some(issuer) + || active.token != expected_token + || !active.can_refresh() + { + return Ok(AlertCursorDelivery::SessionChanged); + } + if let Some(current) = read_alert_cursor(credentials_path, issuer)? + && current >= sequence + { + return Ok(AlertCursorDelivery::AlreadyRecorded(current)); + } + deliver()?; + let path = alert_cursor_path(credentials_path)?; + write_private_json( + &path, + &AlertCursor { + version: ALERT_CURSOR_VERSION, + issuer: issuer.to_string(), + sequence, + }, + "operator alert cursor", + )?; + Ok(AlertCursorDelivery::Delivered(sequence)) +} + fn pending_path(credentials_path: &Path) -> Result { let parent = credentials_path .parent() @@ -228,6 +317,13 @@ fn pending_path(credentials_path: &Path) -> Result { Ok(parent.join("pending-revocations.json")) } +fn alert_cursor_path(credentials_path: &Path) -> Result { + let parent = credentials_path + .parent() + .context("credentials path must have a parent directory")?; + Ok(parent.join("operator-alert-cursor.json")) +} + fn write_private_json(path: &Path, value: &T, description: &str) -> Result<()> { let parent = path .parent() @@ -282,6 +378,29 @@ pub struct CredentialLock { _file: File, } +/// A singleton lock held for the lifetime of one operator alert watcher. +pub struct AlertWatchLock { + _file: File, +} + +impl AlertWatchLock { + pub fn acquire(credentials_path: &Path) -> Result { + let parent = credentials_path + .parent() + .context("credentials path must have a parent directory")?; + create_private_dir(parent)?; + let path = parent.join(".operator-alert-watch.lock"); + let file = open_private_lock_file(&path)?; + match file.try_lock_exclusive() { + Ok(()) => Ok(Self { _file: file }), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + anyhow::bail!("another postil operator alert watcher is already running") + } + Err(error) => Err(error).context("locking operator alert watcher"), + } + } +} + impl CredentialLock { pub async fn acquire(credentials_path: &Path) -> Result { let parent = credentials_path @@ -583,4 +702,184 @@ mod tests { let temp_name = format!(".pending-revocations.json.{}.tmp", std::process::id()); assert!(!pending_path.parent().unwrap().join(temp_name).exists()); } + + #[tokio::test] + #[cfg(unix)] + async fn operator_alert_cursor_is_private_atomic_monotonic_and_issuer_bound() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let credentials_path = dir.path().join("postil").join("credentials.json"); + let stored = sample("2999-01-01T00:00:00.000Z"); + let token = stored.token.clone(); + write(&credentials_path, &stored).unwrap(); + let deliveries = std::cell::Cell::new(0_u32); + assert_eq!( + deliver_alert_with_cursor(&credentials_path, "https://postil.dev", &token, 41, || { + deliveries.set(deliveries.get() + 1); + Ok(()) + }) + .await + .unwrap(), + AlertCursorDelivery::Delivered(41) + ); + assert_eq!( + deliver_alert_with_cursor(&credentials_path, "https://postil.dev", &token, 42, || { + deliveries.set(deliveries.get() + 1); + Ok(()) + }) + .await + .unwrap(), + AlertCursorDelivery::Delivered(42) + ); + assert_eq!( + deliver_alert_with_cursor(&credentials_path, "https://postil.dev", &token, 40, || { + deliveries.set(deliveries.get() + 1); + Ok(()) + }) + .await + .unwrap(), + AlertCursorDelivery::AlreadyRecorded(42) + ); + assert_eq!( + deliver_alert_with_cursor( + &credentials_path, + "https://other.example.test", + &token, + 43, + || { + deliveries.set(deliveries.get() + 1); + Ok(()) + }, + ) + .await + .unwrap(), + AlertCursorDelivery::SessionChanged + ); + let mut replacement = stored.clone(); + replacement.token = "pcli_replacement-access-not-a-real-secret".to_string(); + write(&credentials_path, &replacement).unwrap(); + assert_eq!( + deliver_alert_with_cursor(&credentials_path, "https://postil.dev", &token, 43, || { + deliveries.set(deliveries.get() + 1); + Ok(()) + }) + .await + .unwrap(), + AlertCursorDelivery::SessionChanged + ); + assert_eq!(deliveries.get(), 2); + + assert_eq!( + read_alert_cursor(&credentials_path, "https://postil.dev").unwrap(), + Some(42) + ); + assert_eq!( + read_alert_cursor(&credentials_path, "https://other.example.test").unwrap(), + None + ); + let cursor_path = credentials_path + .parent() + .unwrap() + .join("operator-alert-cursor.json"); + let mode = fs::metadata(&cursor_path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + let temp_name = format!(".operator-alert-cursor.json.{}.tmp", std::process::id()); + assert!(!cursor_path.parent().unwrap().join(temp_name).exists()); + } + + #[tokio::test] + async fn operator_alert_cursor_rejects_out_of_range_sequences() { + let dir = tempfile::tempdir().unwrap(); + let credentials_path = dir.path().join("postil").join("credentials.json"); + assert!( + deliver_alert_with_cursor( + &credentials_path, + "https://postil.dev", + "unused", + i64::MAX as u64 + 1, + || Ok(()) + ) + .await + .is_err() + ); + } + + #[tokio::test] + async fn operator_alert_cursor_compares_and_writes_while_holding_the_login_lock() { + let dir = tempfile::tempdir().unwrap(); + let credentials_path = dir.path().join("postil").join("credentials.json"); + let stored = sample("2999-01-01T00:00:00.000Z"); + let token = stored.token.clone(); + write(&credentials_path, &stored).unwrap(); + deliver_alert_with_cursor(&credentials_path, "https://postil.dev", &token, 41, || { + Ok(()) + }) + .await + .unwrap(); + + let held_lock = CredentialLock::acquire(&credentials_path).await.unwrap(); + let waiting_path = credentials_path.clone(); + let waiting_token = token.clone(); + let deliveries = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let waiting_deliveries = deliveries.clone(); + let waiting_write = tokio::spawn(async move { + deliver_alert_with_cursor( + &waiting_path, + "https://postil.dev", + &waiting_token, + 42, + move || { + waiting_deliveries.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Ok(()) + }, + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(75)).await; + assert!(!waiting_write.is_finished()); + + write_private_json( + &alert_cursor_path(&credentials_path).unwrap(), + &AlertCursor { + version: ALERT_CURSOR_VERSION, + issuer: "https://postil.dev".to_string(), + sequence: 43, + }, + "operator alert cursor", + ) + .unwrap(); + drop(held_lock); + assert_eq!( + waiting_write.await.unwrap().unwrap(), + AlertCursorDelivery::AlreadyRecorded(43) + ); + assert_eq!(deliveries.load(std::sync::atomic::Ordering::Relaxed), 0); + + assert_eq!( + read_alert_cursor(&credentials_path, "https://postil.dev").unwrap(), + Some(43) + ); + } + + #[test] + #[cfg(unix)] + fn operator_alert_watcher_lock_is_private_and_singleton() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let credentials_path = dir.path().join("postil").join("credentials.json"); + let first = AlertWatchLock::acquire(&credentials_path).unwrap(); + assert!(AlertWatchLock::acquire(&credentials_path).is_err()); + let path = credentials_path + .parent() + .unwrap() + .join(".operator-alert-watch.lock"); + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + drop(first); + AlertWatchLock::acquire(&credentials_path).unwrap(); + } } diff --git a/src/lib.rs b/src/lib.rs index 0ddb27a..49a964f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ //! Postil review engine. See README for the product doctrine. pub(crate) mod adjudication; +pub mod alerts; pub(crate) mod api_key; #[cfg(feature = "qualification-candidate")] pub mod attribution; diff --git a/src/llm.rs b/src/llm.rs index 94d730d..4a488ad 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -11,6 +11,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, ensure}; +use hickory_resolver::Resolver; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use serde::Deserialize; use serde_json::json; @@ -844,7 +845,6 @@ const ENDPOINT_AUTH_HEADER_ENV: &str = "POSTIL_ENDPOINT_AUTH_HEADER"; const ENDPOINT_AUTH_VALUE_ENV: &str = "POSTIL_ENDPOINT_AUTH_VALUE"; const ALLOW_PRIVATE_API_BASE_ENV: &str = "POSTIL_ALLOW_PRIVATE_API_BASE"; static PROVIDER_RETRY_JITTER_SEQUENCE: AtomicU64 = AtomicU64::new(0); -#[cfg(feature = "qualification-candidate")] const QUALIFICATION_CAPTURE_API_BASE_ENV: &str = "POSTIL_QUALIFICATION_CAPTURE_API_BASE"; const ALWAYS_MANAGED_HEADERS: &[&str] = &[ "x-api-key", @@ -5213,15 +5213,27 @@ fn env_flag(name: &str) -> bool { } fn qualification_request_api_base(canonical_api_base: &str) -> Result { - #[cfg(feature = "qualification-candidate")] - if crate::config::qualification_candidate_mode() + let qualification_candidate = { + #[cfg(feature = "qualification-candidate")] + { + crate::config::qualification_candidate_mode() + } + #[cfg(not(feature = "qualification-candidate"))] + { + false + } + }; + let release_screen = is_canonical_openrouter_base(canonical_api_base) + && std::env::var_os("POSTIL_BENCH_SCREEN_PROFILE").is_some() + && env_flag("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY"); + if (qualification_candidate || release_screen) && let Some(raw) = std::env::var_os(QUALIFICATION_CAPTURE_API_BASE_ENV) { let raw = raw .into_string() - .map_err(|_| anyhow!("qualification capture API base must be UTF-8"))?; + .map_err(|_| anyhow!("benchmark capture API base must be UTF-8"))?; let url = reqwest::Url::parse(&raw) - .context("qualification capture API base must be an absolute URL")?; + .context("benchmark capture API base must be an absolute URL")?; let loopback = url .host_str() .and_then(|host| host.parse::().ok()) @@ -5233,7 +5245,7 @@ fn qualification_request_api_base(canonical_api_base: &str) -> Result { && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), - "qualification capture API base must be an HTTP loopback URL without credentials, query, or fragment" + "benchmark capture API base must be an HTTP loopback URL without credentials, query, or fragment" ); return Ok(raw.trim_end_matches('/').to_string()); } @@ -5834,6 +5846,34 @@ struct AnthropicUsage { pub(crate) fn secure_http_client(api_base: &str) -> Result { let (hostname, addresses) = resolve_api_endpoint(api_base)?; + build_secure_http_client(&hostname, &addresses) +} + +/// Builds the same DNS-pinned client through Hickory's asynchronous resolver. +/// Alert streams and renewable-login operations use this path so cancellation +/// and their outer deadlines are not defeated by libc `getaddrinfo`. +pub(crate) async fn secure_http_client_async(api_base: &str) -> Result { + let (url, hostname, port) = parse_api_endpoint(api_base)?; + let addresses = if let Ok(address) = hostname.parse::() { + vec![SocketAddr::new(address, port)] + } else { + let resolver = Resolver::builder_tokio() + .context("reading system DNS configuration")? + .build() + .context("building asynchronous DNS resolver")?; + resolver + .lookup_ip(hostname.as_str()) + .await + .with_context(|| format!("model API hostname {hostname:?} could not be resolved"))? + .iter() + .map(|address| SocketAddr::new(address, port)) + .collect() + }; + validate_api_endpoint_addresses(&url, &hostname, &addresses)?; + build_secure_http_client(&hostname, &addresses) +} + +fn build_secure_http_client(hostname: &str, addresses: &[SocketAddr]) -> Result { reqwest::Client::builder() // A system proxy resolves the destination itself and would bypass the // validated, pinned DNS result below while carrying provider secrets. @@ -5844,7 +5884,7 @@ pub(crate) fn secure_http_client(api_base: &str) -> Result { // Connect only to the addresses approved by the single resolution // above. Reqwest retains the URL hostname for TLS SNI/certificate // verification while replacing DNS lookup results with this set. - .resolve_to_addrs(&hostname, &addresses) + .resolve_to_addrs(hostname, addresses) .build() .context("build model provider HTTP client") } @@ -5861,6 +5901,14 @@ fn resolve_api_endpoint_with(api_base: &str, resolver: F) -> Result<(String, where F: FnOnce(&str, u16) -> std::io::Result>, { + let (url, hostname, port) = parse_api_endpoint(api_base)?; + let addresses = resolver(&hostname, port) + .with_context(|| format!("model API hostname {hostname:?} could not be resolved"))?; + validate_api_endpoint_addresses(&url, &hostname, &addresses)?; + Ok((hostname, addresses)) +} + +fn parse_api_endpoint(api_base: &str) -> Result<(reqwest::Url, String, u16)> { let url = reqwest::Url::parse(api_base).context("model API base must be an absolute URL")?; anyhow::ensure!( matches!(url.scheme(), "http" | "https"), @@ -5878,8 +5926,14 @@ where let port = url .port_or_known_default() .context("model API base must include a port for its URL scheme")?; - let addresses = resolver(&hostname, port) - .with_context(|| format!("model API hostname {hostname:?} could not be resolved"))?; + Ok((url, hostname, port)) +} + +fn validate_api_endpoint_addresses( + url: &reqwest::Url, + hostname: &str, + addresses: &[SocketAddr], +) -> Result<()> { anyhow::ensure!( !addresses.is_empty(), "model API hostname {hostname:?} did not resolve to any addresses" @@ -5894,14 +5948,14 @@ where ); } if !allow_private { - for address in &addresses { + for address in addresses { anyhow::ensure!( is_public_ip(address.ip()), "model API hostname {hostname:?} resolved to a private, loopback, link-local, or non-public address" ); } } - Ok((hostname, addresses)) + Ok(()) } fn is_public_ip(address: IpAddr) -> bool { @@ -6896,6 +6950,35 @@ mod tests { assert!(error.to_string().contains("non-public address")); } + #[test] + fn release_screen_capture_preserves_canonical_identity_and_requires_loopback() { + let _lock = env_lock().lock().unwrap(); + let _env = EnvRestore::capture(&[ + QUALIFICATION_CAPTURE_API_BASE_ENV, + "POSTIL_BENCH_SCREEN_PROFILE", + "POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", + ]); + EnvRestore::set("POSTIL_BENCH_SCREEN_PROFILE", "/tmp/profile.json"); + EnvRestore::set("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", "1"); + EnvRestore::set(QUALIFICATION_CAPTURE_API_BASE_ENV, "http://127.0.0.1:4321"); + assert_eq!( + qualification_request_api_base(crate::config::MANAGED_OPENROUTER_API_BASE).unwrap(), + "http://127.0.0.1:4321" + ); + assert_eq!( + qualification_request_api_base("https://models.example/v1").unwrap(), + "https://models.example/v1" + ); + + EnvRestore::set( + QUALIFICATION_CAPTURE_API_BASE_ENV, + "https://models.example/v1", + ); + let error = qualification_request_api_base(crate::config::MANAGED_OPENROUTER_API_BASE) + .expect_err("non-loopback capture must fail closed"); + assert!(error.to_string().contains("HTTP loopback URL")); + } + #[test] fn api_endpoint_resolution_preserves_public_addresses_for_pinning() { let _lock = env_lock().lock().unwrap(); diff --git a/src/login.rs b/src/login.rs index f864d7b..4a218ff 100644 --- a/src/login.rs +++ b/src/login.rs @@ -6,6 +6,7 @@ //! when no explicit key environment variable is set, so a first-time user gets //! working hosted inference with zero configuration. +use std::fmt; use std::path::Path; use std::time::Duration; @@ -15,7 +16,7 @@ use serde::de::DeserializeOwned; use crate::config::normalize_api_base; use crate::credentials::{self, Credentials, PendingRevocation}; -use crate::llm::secure_http_client; +use crate::llm::secure_http_client_async; /// Overrides the Postil web app the device-flow calls target. Distinct from /// `POSTIL_API_BASE`, which (once a token is in hand) points at the @@ -33,6 +34,37 @@ const REFRESH_RESPONSE_MAX_BYTES: usize = 16 * 1024; const REFRESH_REQUEST_TIMEOUT: Duration = Duration::from_secs(20); const REVOCATION_REQUEST_TIMEOUT: Duration = Duration::from_secs(20); +#[derive(Debug)] +enum TransientRefreshError { + Retry, + RetryAfter(u64), +} + +impl fmt::Display for TransientRefreshError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Retry => { + formatter.write_str("could not refresh the stored postil login; try again") + } + Self::RetryAfter(seconds) => write!( + formatter, + "could not refresh the stored postil login; retry after {seconds} seconds" + ), + } + } +} + +impl std::error::Error for TransientRefreshError {} + +pub(crate) fn token_resolution_retry_delay(error: &anyhow::Error) -> Option { + error + .downcast_ref::() + .map(|error| match error { + TransientRefreshError::Retry => Duration::ZERO, + TransientRefreshError::RetryAfter(seconds) => Duration::from_secs(*seconds), + }) +} + fn canonicalize_issuer(value: &str) -> Result { anyhow::ensure!( !value.trim().is_empty(), @@ -76,7 +108,7 @@ fn login_server() -> Result { login_server_override()?.map_or_else(|| canonicalize_issuer(DEFAULT_LOGIN_SERVER), Ok) } -fn stored_issuer(credentials: &Credentials) -> Result { +pub(crate) fn stored_issuer(credentials: &Credentials) -> Result { if let Some(issuer) = credentials.issuer.as_deref() { return canonicalize_issuer(issuer) .context("the stored login issuer is invalid; run `postil login` again"); @@ -128,7 +160,9 @@ fn require_matching_api_base(credentials: &Credentials, resolved_api_base: &str) pub async fn run_login(org: Option) -> Result { let server = login_server()?; - let client = secure_http_client(&server).context("building the postil login HTTP client")?; + let client = secure_http_client_async(&server) + .await + .context("building the postil login HTTP client")?; let path = credentials::default_path()?; login_with(&client, &server, org.as_deref(), &path).await } @@ -151,7 +185,7 @@ enum ClientMode<'a> { } impl ClientMode<'_> { - fn for_issuer(self, issuer: &str) -> Result { + async fn for_issuer(self, issuer: &str) -> Result { match self { #[cfg(test)] Self::Provided(client) => Ok(client.clone()), @@ -159,7 +193,8 @@ impl ClientMode<'_> { client, issuer: client_issuer, } if client_issuer == issuer => Ok(client.clone()), - Self::Secure | Self::Reuse { .. } => secure_http_client(issuer) + Self::Secure | Self::Reuse { .. } => secure_http_client_async(issuer) + .await .context("building an HTTP client for stored login issuer"), } } @@ -464,6 +499,45 @@ pub(crate) async fn resolve_stored_token( result } +#[derive(Debug)] +pub(crate) struct StoredAlertSession { + pub issuer: String, + pub token: String, +} + +/// Returns an issuer and renewable access token from one coherent stored-login +/// generation. A concurrent login replacement retries instead of pairing the +/// replacement token with the prior issuer. +pub(crate) async fn resolve_stored_alert_session( + credentials_path: &Path, +) -> Result> { + for _ in 0..3 { + let Some(snapshot) = credentials::read(credentials_path)? else { + return Ok(None); + }; + let Some(token) = resolve_stored_token(credentials_path, &snapshot.api_base).await? else { + return Ok(None); + }; + let Some(current) = credentials::read(credentials_path)? else { + continue; + }; + if current.api_base != snapshot.api_base || current.token != token { + continue; + } + anyhow::ensure!( + current.version == credentials::CREDENTIALS_VERSION + && current.issuer.is_some() + && current.can_refresh(), + "operator alert notifications require a renewable login; run `postil login` again" + ); + return Ok(Some(StoredAlertSession { + issuer: stored_issuer(¤t)?, + token, + })); + } + anyhow::bail!("the stored postil login changed while connecting; try again") +} + #[cfg(test)] async fn resolve_stored_token_with( client: &reqwest::Client, @@ -522,7 +596,7 @@ async fn resolve_stored_token_with_mode( anyhow::bail!("the stored postil login credential expired; run `postil login` again"); }; - let client = client_mode.for_issuer(&issuer)?; + let client = client_mode.for_issuer(&issuer).await?; let refreshed = refresh_token_with(&client, &issuer, refresh_token).await?; let replacement = Credentials { version: credentials::CREDENTIALS_VERSION, @@ -626,7 +700,7 @@ async fn revoke(client_mode: ClientMode<'_>, revocation: &PendingRevocation) -> let Ok(issuer) = canonicalize_issuer(&revocation.issuer) else { return false; }; - let Ok(client) = client_mode.for_issuer(&issuer) else { + let Ok(client) = client_mode.for_issuer(&issuer).await else { return false; }; let logout_url = format!("{issuer}/api/cli/logout"); @@ -733,8 +807,8 @@ async fn refresh_token_with( .send(), ) .await - .map_err(|_| anyhow!("could not refresh the stored postil login; try again"))? - .map_err(|_| anyhow!("could not refresh the stored postil login; try again"))?; + .map_err(|_| TransientRefreshError::Retry)? + .map_err(|_| TransientRefreshError::Retry)?; let status = response.status(); if status.as_u16() == 429 { if let Some(retry_after) = response @@ -747,14 +821,17 @@ async fn refresh_token_with( && value.parse::().is_ok() }) { - anyhow::bail!( - "could not refresh the stored postil login; retry after {retry_after} seconds" - ); + return Err(TransientRefreshError::RetryAfter( + retry_after + .parse() + .expect("numeric Retry-After was validated above"), + ) + .into()); } - anyhow::bail!("could not refresh the stored postil login; try again"); + return Err(TransientRefreshError::Retry.into()); } if status.is_server_error() || status.as_u16() == 408 { - anyhow::bail!("could not refresh the stored postil login; try again"); + return Err(TransientRefreshError::Retry.into()); } if !status.is_success() { anyhow::bail!("the stored postil login can no longer be renewed; run `postil login` again"); @@ -765,7 +842,7 @@ async fn refresh_token_with( anyhow::bail!("the postil refresh response was invalid; run `postil login` again") } Ok(Err(RefreshResponseError::Transport)) | Err(_) => { - anyhow::bail!("could not refresh the stored postil login; try again") + Err(TransientRefreshError::Retry.into()) } } } @@ -1441,6 +1518,38 @@ mod tests { credentials } + #[tokio::test] + async fn alert_session_returns_one_renewable_issuer_token_generation() { + let dir = tempfile::tempdir().unwrap(); + let credentials_path = dir.path().join("postil").join("credentials.json"); + let stored = stored_credentials("2999-01-01T00:00:00.000Z"); + credentials::write(&credentials_path, &stored).unwrap(); + + let session = resolve_stored_alert_session(&credentials_path) + .await + .unwrap() + .unwrap(); + assert_eq!(session.issuer, DEFAULT_LOGIN_SERVER); + assert_eq!(session.token, stored.token); + } + + #[tokio::test] + async fn alert_session_rejects_an_access_only_login() { + let dir = tempfile::tempdir().unwrap(); + let credentials_path = dir.path().join("postil").join("credentials.json"); + let mut stored = stored_credentials("2999-01-01T00:00:00.000Z"); + stored.version = credentials::LEGACY_CREDENTIALS_VERSION; + stored.issuer = None; + stored.refresh_token = None; + stored.refresh_expires_at = None; + credentials::write(&credentials_path, &stored).unwrap(); + + let error = resolve_stored_alert_session(&credentials_path) + .await + .expect_err("operator notifications require renewable credentials"); + assert!(error.to_string().contains("renewable login")); + } + fn write_legacy_credentials(path: &Path, credentials: &Credentials) { std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write(path, serde_json::to_string(credentials).unwrap()).unwrap(); @@ -1586,6 +1695,7 @@ mod tests { .await .expect_err("a failed refresh must fail closed"); assert!(error.to_string().contains("try again")); + assert_eq!(token_resolution_retry_delay(&error), Some(Duration::ZERO)); assert_eq!( credentials::read(&credentials_path).unwrap().unwrap(), original @@ -1637,6 +1747,7 @@ mod tests { .await .expect_err("a replayed refresh token must require login"); assert!(error.to_string().contains("postil login")); + assert_eq!(token_resolution_retry_delay(&error), None); assert!(!error.to_string().contains("refresh replay details")); assert_eq!( credentials::read(&credentials_path).unwrap().unwrap(), @@ -1886,6 +1997,10 @@ mod tests { .expect_err("rate-limited refreshes must fail closed without relogin"); assert!(error.to_string().contains("retry after 3527 seconds")); assert!(!error.to_string().contains("run `postil login` again")); + assert_eq!( + token_resolution_retry_delay(&error), + Some(Duration::from_secs(3527)) + ); assert_eq!( credentials::read(&credentials_path).unwrap().unwrap(), original @@ -1919,6 +2034,7 @@ mod tests { .await .expect_err("an unusable Retry-After must fail safely"); assert!(error.to_string().contains("try again")); + assert_eq!(token_resolution_retry_delay(&error), Some(Duration::ZERO)); assert!(!error.to_string().contains("retry after")); assert_eq!( credentials::read(&credentials_path).unwrap().unwrap(), diff --git a/src/main.rs b/src/main.rs index c25e224..073b7b5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ use postil_cli::config::{ default_scorer_reasoning_effort, qualification_metadata, starter_config, }; use postil_cli::review::{ForgeKind, ReviewArgs}; -use postil_cli::{doctor, hook, login, plan, review}; +use postil_cli::{alerts, doctor, hook, login, plan, review}; #[tokio::main] async fn main() { @@ -27,6 +27,7 @@ async fn main() { async fn dispatch(cli: Cli) -> anyhow::Result { match cli.command { + Command::AlertWatch { once, probe } => alerts::run_watch(once, probe).await, Command::Capabilities { publication_plan_contract, } => { @@ -152,6 +153,10 @@ async fn dispatch(cli: Cli) -> anyhow::Result { println!( " OpenAI-compatible endpoints accept any non-empty endpoint model ID and pass it unchanged; OpenRouter commonly uses provider/model." ); + println!( + " Recommended OpenRouter starting point: {} (the embedded default); `postil doctor` verifies current provider availability.", + postil_cli::config::default_model() + ); println!( " Native Anthropic endpoints accept any non-empty Anthropic endpoint model ID and pass it unchanged, such as claude-* IDs." ); @@ -174,7 +179,10 @@ async fn dispatch(cli: Cli) -> anyhow::Result { } ); } else { - println!(" Hosted qualified model IDs: none (no embedded qualified profile)"); + println!( + " Hosted model selection is service-controlled; `postil login` does not require a model setting." + ); + println!(" This binary contains no standalone hosted qualification profile."); } println!("\nCheck the configured endpoint and model: postil doctor"); println!("Override once: postil review --model provider/model"); diff --git a/src/output.rs b/src/output.rs index b16afb4..959a4ae 100644 --- a/src/output.rs +++ b/src/output.rs @@ -152,16 +152,18 @@ fn csv_field(field: String) -> String { } } -pub fn print_pretty(envelope: &Envelope) { +pub fn print_pretty(envelope: &Envelope, compact: bool) { let color = std::io::stderr().is_terminal() && std::env::var_os("NO_COLOR").is_none(); let mut out = String::new(); if envelope.silent { - out.push_str(&paint( - color, - "✓ postil: no merge-relevant findings. Staying silent.\n", - Paint::Green, - )); + if !compact { + out.push_str(&paint( + color, + "✓ postil: no merge-relevant findings. Staying silent.\n", + Paint::Green, + )); + } } else { if !envelope.summary.is_empty() { out.push_str(&format!("{}\n\n", sanitize(&envelope.summary))); @@ -210,19 +212,21 @@ pub fn print_pretty(envelope: &Envelope) { envelope.counts.suppressed )); } - if let Some(coverage) = &envelope.review_coverage { - out.push_str(&render_review_coverage(coverage)); + if !compact { + if let Some(coverage) = &envelope.review_coverage { + out.push_str(&render_review_coverage(coverage)); + } + out.push_str(&render_repository_search(&envelope.repository_search)); + let gate = if envelope.gate.failing { + paint(color, "gate: failing", Paint::Red) + } else { + paint(color, "gate: passing", Paint::Green) + }; + out.push_str(&format!( + "{gate} (fail-on: {}) model: {}\n", + envelope.gate.fail_on, envelope.model_used + )); } - out.push_str(&render_repository_search(&envelope.repository_search)); - let gate = if envelope.gate.failing { - paint(color, "gate: failing", Paint::Red) - } else { - paint(color, "gate: passing", Paint::Green) - }; - out.push_str(&format!( - "{gate} (fail-on: {}) model: {}\n", - envelope.gate.fail_on, envelope.model_used - )); eprint!("{out}"); } diff --git a/src/progress.rs b/src/progress.rs index daad824..d362da4 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -66,6 +66,10 @@ pub fn telemetry(message: Arguments<'_>) { } } +pub fn compact_human_output() -> bool { + SUPPRESS_TELEMETRY.load(Ordering::Relaxed) +} + /// Keep safety-relevant notices visible even while detailed telemetry is /// collapsed into an interactive progress line. pub fn notice(message: Arguments<'_>) { diff --git a/src/review.rs b/src/review.rs index 86cf1c3..42779cc 100644 --- a/src/review.rs +++ b/src/review.rs @@ -235,17 +235,6 @@ fn review_batch_validation_reason( ), }); } - if crate::repository_search::prose_requires_repository_search(finding) - && finding.repository_claim.is_none() - { - return Some(ReviewBatchValidationReason { - category: "repositoryClaim", - repair_detail: format!( - "finding at {}:{} makes a repository-wide absence or mismatch claim without a bounded repositoryContext declaration", - finding.path, finding.line - ), - }); - } if let Some(claim) = finding.repository_claim.as_ref() && !crate::repository_search::claim_is_valid(claim) { @@ -3078,7 +3067,7 @@ async fn finish( && (args.resolved_output_format().is_none() || args.output_file.is_some()) { let _progress_suspension = crate::progress::suspend_for_output(); - output::print_pretty(&envelope); + output::print_pretty(&envelope, crate::progress::compact_human_output()); } let duplicate_of_baseline = load_baseline(args) @@ -5289,7 +5278,7 @@ mod tests { } #[test] - fn batch_validation_requires_typed_queries_for_universal_repository_claims() { + fn batch_validation_defers_unstructured_repository_claims_to_suppression() { let annotated = "### src/lib.rs\n@@ fixture @@\n 7 + changed();\n"; let mut finding = finding( "src/lib.rs", @@ -5298,8 +5287,10 @@ mod tests { ); finding.evidence = Some("changed();".to_string()); - let reason = review_batch_validation_reason(&finding, annotated, None).unwrap(); - assert_eq!(reason.category, "repositoryClaim"); + assert_eq!( + review_batch_validation_reason(&finding, annotated, None), + None + ); finding.repository_claim = Some(crate::envelope::RepositoryClaim { kind: crate::envelope::RepositoryClaimKind::Absence, @@ -5313,6 +5304,17 @@ mod tests { review_batch_validation_reason(&finding, annotated, None), None ); + + finding + .repository_claim + .as_mut() + .unwrap() + .identifiers + .clear(); + assert_eq!( + review_batch_validation_reason(&finding, annotated, None).map(|reason| reason.category), + Some("repositoryClaim") + ); } #[test] diff --git a/tests/e2e.rs b/tests/e2e.rs index 4003721..95e6521 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -6535,8 +6535,33 @@ async fn local_review_reports_grounded_finding_and_gates() { ); } +#[tokio::test] +async fn bare_review_selects_a_nonempty_local_change_and_reaches_the_provider() { + let server = MockServer::start().await; + mock_review(&server, json!([])).await; + let directory = tempfile::tempdir().unwrap(); + initialize_staged_repository(directory.path()); + + let output = postil() + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("POSTIL_DISABLE_SCORER", "1") + .env_remove("REVIEW_MODEL") + .env_remove("REVIEW_MODEL_CASCADE") + .args(["review", "--output", "json"]) + .assert() + .success(); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + assert_eq!(envelope["silent"], true); + assert_eq!(envelope["modelUsed"], "openai/gpt-5.6-luna"); + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert!(String::from_utf8_lossy(&requests[0].body).contains("exec_query(&token)")); +} + #[test] -fn review_without_an_explicit_model_exits_before_provider_access() { +fn review_with_the_embedded_model_reports_a_missing_provider_credential() { let dir = tempfile::tempdir().unwrap(); let diff = write_diff(dir.path()); let out = postil() @@ -6613,8 +6638,8 @@ fn bare_review_clears_progress_before_pretty_output_in_a_pty() { assert!(output.status.success()); let rendered = String::from_utf8_lossy(&output.stdout); let summary = rendered - .find("✓ postil: no merge-relevant findings") - .expect("pretty output was not rendered"); + .find("postil: review complete; no findings") + .expect("completion output was not rendered"); let before_summary = &rendered[..summary]; let spinner = before_summary .rfind("Reviewing changes...") @@ -6626,6 +6651,8 @@ fn bare_review_clears_progress_before_pretty_output_in_a_pty() { clear > spinner, "pretty output began before the progress line was cleared: {rendered:?}" ); + assert!(!rendered.contains("repository search:"), "{rendered:?}"); + assert!(!rendered.contains("gate: passing"), "{rendered:?}"); } #[cfg(unix)] @@ -6677,9 +6704,11 @@ fn file_artifacts_keep_human_progress_and_pretty_output_in_a_pty() { let rendered = String::from_utf8_lossy(&output.stdout); assert!(rendered.contains("Reviewing changes"), "{rendered:?}"); assert!( - rendered.contains("✓ postil: no merge-relevant findings"), + rendered.contains("postil: review complete; no findings"), "{rendered:?}" ); + assert!(!rendered.contains("repository search:"), "{rendered:?}"); + assert!(!rendered.contains("gate: passing"), "{rendered:?}"); } serde_json::from_slice::(&std::fs::read(envelope).unwrap()).unwrap(); @@ -6746,6 +6775,35 @@ async fn interactive_progress_controls_animation_separately_from_telemetry() { "machine review omitted GitHub telemetry: {telemetry:?}" ); + let ci = isolated_script( + &binary, + &[ + "review".to_string(), + "--repo".to_string(), + "acme/api".to_string(), + "--pr".to_string(), + "7".to_string(), + ], + ) + .current_dir(dir.path()) + .env("TERM", "xterm") + .env("CI", "true") + .env("GITHUB_API_URL", server.uri()) + .env("GITHUB_TOKEN", "gh-test-token") + .env_remove("POSTIL_HOSTED_MODE") + .env_remove("RUST_LOG") + .env_remove("POSTIL_DEBUG") + .env_remove("NO_COLOR") + .output() + .unwrap(); + assert!(ci.status.success()); + let ci = String::from_utf8_lossy(&ci.stdout); + assert!( + ci.contains("postil: github operation="), + "CI review omitted operational telemetry: {ci:?}" + ); + assert!(!ci.contains("Reviewing changes"), "{ci:?}"); + for (arguments, no_progress_environment) in [ ( vec![ @@ -7030,13 +7088,17 @@ fn models_and_config_explain_the_embedded_default() { assert!(models.contains("does not maintain a fixed local model-ID allowlist")); assert!(models.contains("OpenAI-compatible endpoints accept any non-empty endpoint model ID")); assert!(models.contains("OpenRouter commonly uses provider/model")); + assert!(models.contains("Recommended OpenRouter starting point: openai/gpt-5.6-luna")); + assert!(models.contains("postil doctor` verifies current provider availability")); assert!( models.contains( "Native Anthropic endpoints accept any non-empty Anthropic endpoint model ID" ) ); assert!(models.contains("does not mean the model is hosted-qualified")); - assert!(models.contains("Hosted qualified model IDs: none (no embedded qualified profile)")); + assert!(models.contains("Hosted model selection is service-controlled")); + assert!(models.contains("postil login` does not require a model setting")); + assert!(models.contains("no standalone hosted qualification profile")); assert!(models.contains("max|xhigh|high|medium|low|minimal|none")); assert!(models.contains("postil doctor")); assert!(models.contains("postil review --model provider/model")); @@ -13370,6 +13432,69 @@ async fn fresh_unresolved_repository_claims_are_suppressed() { } } +#[tokio::test] +async fn unstructured_repository_claim_is_suppressed_without_discarding_the_review() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(AllUnresolvedAdjudicator) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + mock_review( + &server, + json!([{ + "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", + "confidence": 0.99, "title": "Caller support is absent", + "body": "No other caller accepts this value.", + "evidence": "exec_query(&token);" + }]), + ) + .await; + let directory = tempfile::tempdir().unwrap(); + let diff = write_diff(directory.path()); + + let output = postil() + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("POSTIL_DISABLE_SCORER", "1") + .args(["review", "--diff-file"]) + .arg(&diff) + .args(["--output", "json"]) + .assert() + .success(); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + + assert_eq!(envelope["findings"], json!([])); + assert_eq!(envelope["counts"]["suppressed"], 1); + assert_eq!( + envelope["suppressedFindings"][0]["reason"], + "repositoryClaimUnsupported" + ); + assert_eq!(envelope["gate"]["failing"], false); + assert!( + envelope["modelIncidents"] + .as_array() + .is_none_or(Vec::is_empty) + ); + assert!( + envelope["modelUsage"] + .as_array() + .unwrap() + .iter() + .all(|usage| usage["phase"] == "initial") + ); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + assert!( + requests + .iter() + .all(|request| { !String::from_utf8_lossy(&request.body).contains("[Correction]") }) + ); +} + #[tokio::test] async fn full_rereview_rejects_exhausted_baseline_adjudication_capacity_before_provider_contact() { let server = MockServer::start().await;