diff --git a/.github/workflows/benchmark-baseline.yml b/.github/workflows/benchmark-baseline.yml new file mode 100644 index 00000000..4688b95d --- /dev/null +++ b/.github/workflows/benchmark-baseline.yml @@ -0,0 +1,206 @@ +name: Benchmark Gate Calibration and Baseline Capture + +on: + workflow_dispatch: + inputs: + mode: + description: Calibration uses two ten-sample replicas; capture uses one five-sample profile. + required: true + type: choice + options: + - calibration + - capture + source_sha: + description: Confirm the exact protected-main dispatch SHA to build and measure. + required: true + type: string + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + COLDKEEP_AES_GCM_FIXTURE_HEX: ${{ format('{0}{1}{0}{1}{0}{1}{0}{1}', '01234567', '89abcdef') }} + POSTGRES_IMAGE_DIGEST: sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 + +jobs: + authorize: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Validate trusted benchmark source + env: + SOURCE_SHA: ${{ inputs.source_sha }} + TRUSTED_REF: ${{ github.ref }} + TRUSTED_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [[ "${TRUSTED_REF}" != "refs/heads/main" ]]; then + echo "benchmark calibration must be dispatched from refs/heads/main" >&2 + exit 2 + fi + if ! [[ "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "source_sha must be a full lowercase commit SHA" >&2 + exit 2 + fi + if [[ "${SOURCE_SHA}" != "${TRUSTED_SHA}" ]]; then + echo "source_sha must equal the trusted workflow dispatch SHA" >&2 + exit 2 + fi + + sample: + needs: authorize + runs-on: ubuntu-24.04 + timeout-minutes: 30 + services: + postgres: + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 + env: + POSTGRES_USER: coldkeep + POSTGRES_PASSWORD: coldkeep + POSTGRES_DB: coldkeep + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U coldkeep -d coldkeep" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + fail-fast: false + matrix: + compression: [none, zstd] + workers: [1, 4] + replicate: [1, 2] + + steps: + - name: Checkout trusted source + if: ${{ inputs.mode == 'calibration' || matrix.replicate == 1 }} + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify trusted checkout + if: ${{ inputs.mode == 'calibration' || matrix.replicate == 1 }} + env: + TRUSTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "${TRUSTED_SHA}" + + - name: Setup exact Go toolchain + if: ${{ inputs.mode == 'calibration' || matrix.replicate == 1 }} + uses: actions/setup-go@v6 + with: + go-version: '1.25.12' + cache: false + + - name: Build benchmark binary once + if: ${{ inputs.mode == 'calibration' || matrix.replicate == 1 }} + run: go build -o coldkeep ./cmd/coldkeep + + - name: Capture fixed benchmark samples + if: ${{ inputs.mode == 'calibration' || matrix.replicate == 1 }} + env: + SOURCE_SHA: ${{ github.sha }} + CAPTURE_MODE: ${{ inputs.mode }} + COLDKEEP_DB_AUTO_BOOTSTRAP: true + COLDKEEP_CODEC: aes-gcm + COLDKEEP_COMPRESSION: ${{ matrix.compression }} + COLDKEEP_KEY: ${{ env.COLDKEEP_AES_GCM_FIXTURE_HEX }} + COLDKEEP_CONTAINER_LOCK_RETRY_ATTEMPTS: 12 + COLDKEEP_CONTAINER_LOCK_RETRY_BASE_WAIT_MS: 15 + COLDKEEP_CONTAINER_LOCK_RETRY_MAX_WAIT_MS: 900 + DB_HOST: 127.0.0.1 + DB_PORT: 5432 + DB_USER: coldkeep + DB_PASSWORD: coldkeep + DB_NAME: coldkeep + DB_SSLMODE: disable + run: | + sample_count=10 + if [ "${CAPTURE_MODE}" = "capture" ]; then + sample_count=5 + fi + profile="${{ matrix.compression }}-w${{ matrix.workers }}-r${{ matrix.replicate }}" + postgres_version="$(docker exec '${{ job.services.postgres.id }}' postgres --version)" + python3 scripts/benchmark_gate.py sample \ + --binary ./coldkeep \ + --output-dir "evidence/${profile}" \ + --compression '${{ matrix.compression }}' \ + --workers '${{ matrix.workers }}' \ + --dataset ci-stable-v1 \ + --warmups 1 \ + --samples "${sample_count}" \ + --source-commit "${SOURCE_SHA}" \ + --go-version "$(go version)" \ + --postgres-version "${postgres_version}" \ + --database-image-digest "${POSTGRES_IMAGE_DIGEST}" + + - name: Upload immutable sample evidence + if: ${{ always() && (inputs.mode == 'calibration' || matrix.replicate == 1) }} + uses: actions/upload-artifact@v7 + with: + name: benchmark-${{ inputs.mode }}-${{ matrix.compression }}-w${{ matrix.workers }}-r${{ matrix.replicate }} + path: evidence/${{ matrix.compression }}-w${{ matrix.workers }}-r${{ matrix.replicate }} + if-no-files-found: error + + calibration: + if: ${{ inputs.mode == 'calibration' }} + runs-on: ubuntu-24.04 + needs: sample + timeout-minutes: 10 + steps: + - name: Checkout calibration harness + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download calibration evidence + uses: actions/download-artifact@v8 + with: + pattern: benchmark-calibration-* + path: ${{ runner.temp }}/benchmark-calibration-input + + - name: Evaluate the fixed calibration matrix + run: | + mapfile -t aggregates < <( + find "${RUNNER_TEMP}/benchmark-calibration-input" \ + -mindepth 2 -maxdepth 2 -name aggregate.json -print | sort + ) + args=() + for aggregate in "${aggregates[@]}"; do + artifact="$(basename "$(dirname "${aggregate}")")" + profile="${artifact#benchmark-calibration-}" + args+=(--aggregate "${profile}=${aggregate}") + done + python3 - "${GITHUB_SHA}" "${aggregates[@]}" <<'PY' + import json + import pathlib + import sys + + expected = sys.argv[1] + for raw_path in sys.argv[2:]: + path = pathlib.Path(raw_path) + with path.open(encoding="utf-8") as handle: + report = json.load(handle) + actual = report.get("provenance", {}).get("source_commit") + if actual != expected: + raise SystemExit( + f"{path}: source_commit {actual!r} does not match trusted SHA {expected}" + ) + PY + python3 scripts/benchmark_gate.py calibrate \ + "${args[@]}" \ + --thresholds benchmarks/v1.9/regression-thresholds.yaml \ + --output calibration-report.json + + - name: Upload calibration decision + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-calibration-decision + path: calibration-report.json + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05bf8e7a..4bc8f8b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,8 @@ jobs: - name: Check formatting run: | - unformatted=$(gofmt -l $(git ls-files '*.go')) + mapfile -t go_files < <(git ls-files '*.go') + unformatted=$(gofmt -l "${go_files[@]}") if [ -n "$unformatted" ]; then echo "Unformatted Go files detected:" echo "$unformatted" @@ -172,6 +173,64 @@ jobs: go test -race -count=1 -short -json ./tests/integration/... | tee "$json_file" status=${PIPESTATUS[0]} + if [ "$status" -eq 0 ]; then + python3 - "$json_file" "${{ matrix.codec }}" <<'PY' + import json + import re + import sys + + json_file, codec = sys.argv[1:] + package = "github.com/franchoy/coldkeep/tests/integration" + expected = [ + r"^TestRoundTripStoreRestore$", + ] + if codec == "plain": + expected.extend([ + r"^TestRemoveWithSharedChunksRefCount$", + r"^TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState$", + ]) + + events = [] + malformed = [] + try: + with open(json_file, encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, start=1): + try: + events.append(json.loads(raw_line)) + except json.JSONDecodeError as exc: + malformed.append(f"line {line_number}: {exc}") + except OSError as exc: + print(f"required execution-proof failure: cannot read {json_file}: {exc}", file=sys.stderr) + sys.exit(1) + + failures = [] + if not events: + failures.append("JSON evidence file contained no events") + if malformed: + failures.append("malformed JSON evidence: " + "; ".join(malformed[:3])) + + package_events = [event for event in events if event.get("Package") == package] + for pattern in expected: + matching = [ + event for event in package_events + if re.search(pattern, event.get("Test", "")) + ] + skipped = [event.get("Test", "") for event in matching if event.get("Action") == "skip"] + passed = [event.get("Test", "") for event in matching if event.get("Action") == "pass"] + if skipped: + failures.append(f"required test skipped: selector={pattern} tests={skipped}") + if not passed: + failures.append(f"required pass missing: package={package} selector={pattern}") + + if failures: + print("required execution-proof failure:", file=sys.stderr) + for failure in failures: + print(f"- {failure}", file=sys.stderr) + sys.exit(1) + PY + status=$? + fi + if [ "$status" -ne 0 ]; then echo "correctness-matrix failed for codec=${{ matrix.codec }}" first_test_fail=$(grep '"Action":"fail"' "$json_file" | grep '"Test":' | head -n 1 || true) @@ -202,6 +261,150 @@ jobs: exit "$status" + - name: Run required PostgreSQL internal package contracts + if: ${{ matrix.codec == 'plain' }} + env: + COLDKEEP_TEST_DB: 1 + COLDKEEP_DB_AUTO_BOOTSTRAP: true + DB_HOST: 127.0.0.1 + DB_PORT: 5432 + DB_USER: coldkeep + DB_PASSWORD: coldkeep + DB_NAME: coldkeep + DB_SSLMODE: disable + run: | + set -o pipefail + output_file="${RUNNER_TEMP}/postgres-internal-contracts.json" + + go test -race -count=1 -json \ + ./internal/testutil/backendtest \ + ./internal/catalog \ + ./internal/db \ + ./internal/engine \ + ./internal/maintenance \ + ./internal/container \ + | tee "$output_file" + + python3 - "$output_file" <<'PY' + import json + import re + import sys + + output_file = sys.argv[1] + expected = { + "github.com/franchoy/coldkeep/internal/testutil/backendtest": [ + r"^TestForEachSQLiteFixture/postgres$", + ], + "github.com/franchoy/coldkeep/internal/catalog": [ + r"^TestCatalogContractFindLogicalFileAcrossBackends/postgres$", + r"^TestCatalogContractFindPhysicalFilesAcrossBackends/postgres$", + r"^TestCatalogContractFindSnapshotAcrossBackends/postgres$", + r"^TestCatalogContractListSnapshotsAcrossBackends/postgres$", + r"^TestCatalogContractLoadReachabilityRootsAcrossBackends/postgres$", + r"^TestCatalogContractDeferredMethodsAcrossBackends/postgres$", + ], + "github.com/franchoy/coldkeep/internal/db": [ + r"^TestEnsurePostgresSchemaAutoMigratesVersionElevenToTwelve$", + r"^TestSCH001AndSCH002BootstrapVersionAndIdempotency/postgres$", + r"^TestSCH003CurrentSchemaMetadata/postgres$", + r"^TestSCH004MinimalCatalogUsability/postgres$", + r"^TestSCH005CriticalUniqueness/postgres$", + r"^TestSCH006CriticalForeignKeys/postgres$", + r"^TestSCH007NullableAndDefaultSemantics/postgres$", + r"^TestSCH009PostgresVersionElevenAutoMigration/postgres$", + r"^TestSCH010AndSCH011MetadataBoundaries/postgres$", + r"^TestBackendTransactionCommitRollbackAcrossBackends/postgres$", + r"^TestBackendForUpdateLockReleaseAcrossBackends/postgres$", + r"^TestBackendNowaitAndSkipLockedAcrossBackends/postgres$", + r"^TestBackendBlockedLockCancellationAcrossBackends/postgres$", + r"^TestMutationRowsAffectedContractAcrossBackends/postgres$", + ], + "github.com/franchoy/coldkeep/internal/engine": [ + r"^TestRemoveByIDPostgresPreservesSharedChunks$", + r"^TestEngineReadStatsAndInspectAcrossBackends/postgres$", + r"^TestEngineReadSnapshotViewsAcrossBackends/postgres$", + r"^TestEngineReadVerifyAcrossBackends/postgres$", + r"^TestEngineReadContextAndErrorsAcrossBackends/postgres$", + r"^TestEngineSnapshotSelectorsAcrossBackends/postgres$", + r"^TestEngineSnapshotSelectorErrorsAcrossBackends/postgres$", + r"^TestEngineMutationStoreRemoveAcrossBackends/postgres$", + r"^TestEngineMutationSnapshotLifecycleAcrossBackends/postgres$", + r"^TestEngineMutationRestoreAcrossBackends/postgres$", + r"^TestEngineMutationErrorsAcrossBackends/postgres$", + r"^TestEngineGCDryRunAcrossBackends/postgres$", + ], + "github.com/franchoy/coldkeep/internal/maintenance": [ + r"^TestGCAdvisoryLockUsesDedicatedSessionAndReleases$", + r"^TestRunGCReleasesAdvisoryLockAfterOperationFailure$", + r"^TestRunGCAdvisoryCleanupFailureReturnsErrorAndDiscardsSession$", + r"^TestRunGCLiveRefusesSingleConnectionPool$", + ], + "github.com/franchoy/coldkeep/internal/container": [ + r"^TestContainerRowLockIntegrationAcrossBackends/postgres$", + ], + } + + raw_lines = [] + events = [] + malformed = [] + try: + with open(output_file, encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, start=1): + raw_lines.append(raw_line.rstrip("\n")) + try: + events.append(json.loads(raw_line)) + except json.JSONDecodeError as exc: + malformed.append(f"line {line_number}: {exc}") + except OSError as exc: + print(f"PostgreSQL execution-proof failure: cannot read {output_file}: {exc}", file=sys.stderr) + sys.exit(1) + + failures = [] + if not events: + failures.append("JSON evidence file contained no events") + if malformed: + failures.append("malformed JSON evidence: " + "; ".join(malformed[:3])) + + observed = {} + skipped = {} + for event in events: + package = event.get("Package", "") + test = event.get("Test", "") + action = event.get("Action", "") + if package in expected and test: + if re.search("postgres", test, re.IGNORECASE): + observed.setdefault(package, []).append(f"{action}:{test}") + if action == "skip": + skipped.setdefault(package, []).append(test) + + for package, patterns in expected.items(): + package_events = [event for event in events if event.get("Package") == package] + if not package_events: + failures.append(f"expected package produced no JSON events: {package}") + continue + for pattern in patterns: + passes = [ + event.get("Test", "") for event in package_events + if event.get("Action") == "pass" and re.search(pattern, event.get("Test", "")) + ] + if not passes: + failures.append(f"expected PostgreSQL pass missing: package={package} selector={pattern}") + + if failures: + print("PostgreSQL execution-proof failure:", file=sys.stderr) + for failure in failures: + print(f"- {failure}", file=sys.stderr) + for package, patterns in expected.items(): + print(f"expected package: {package}", file=sys.stderr) + print(f"expected selectors: {', '.join(patterns)}", file=sys.stderr) + print(f"observed PostgreSQL-related tests: {observed.get(package, [])}", file=sys.stderr) + print(f"skipped tests: {skipped.get(package, [])}", file=sys.stderr) + print("last 80 JSON lines:", file=sys.stderr) + for line in raw_lines[-80:]: + print(line, file=sys.stderr) + sys.exit(1) + PY + - name: Upload correctness-matrix diagnostic JSON if: failure() uses: actions/upload-artifact@v7 @@ -450,8 +653,65 @@ jobs: echo "diag_dir=$diag_dir" >> "$GITHUB_OUTPUT" echo "log_file=$log_file" >> "$GITHUB_OUTPUT" - go test -race -count=1 ./tests/adversarial/... 2>&1 | tee "$log_file" + go test -race -count=1 -json ./tests/adversarial/... 2>&1 | tee "$log_file" status=${PIPESTATUS[0]} + + if [ "$status" -eq 0 ]; then + python3 - "$log_file" <<'PY' + import json + import re + import sys + + log_file = sys.argv[1] + package = "github.com/franchoy/coldkeep/tests/adversarial" + expected = [ + r"^TestAdversarialG6IndependentProcessRepositoryContention/plain$", + r"^TestAdversarialG6IndependentProcessRepositoryContention/aes-gcm$", + r"^TestAdversarialG6KilledLeaseHolderReleasesRepository$", + r"^TestAdversarialG6LiveGCExcludesIndependentStoreProcess$", + ] + + events = [] + malformed = [] + try: + with open(log_file, encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, start=1): + try: + events.append(json.loads(raw_line)) + except json.JSONDecodeError as exc: + malformed.append(f"line {line_number}: {exc}") + except OSError as exc: + print(f"required execution-proof failure: cannot read {log_file}: {exc}", file=sys.stderr) + sys.exit(1) + + failures = [] + if not events: + failures.append("JSON evidence file contained no events") + if malformed: + failures.append("malformed JSON evidence: " + "; ".join(malformed[:3])) + + package_events = [event for event in events if event.get("Package") == package] + for pattern in expected: + matching = [ + event for event in package_events + if re.search(pattern, event.get("Test", "")) + ] + skipped = [event.get("Test", "") for event in matching if event.get("Action") == "skip"] + passed = [event.get("Test", "") for event in matching if event.get("Action") == "pass"] + if skipped: + failures.append(f"required test skipped: selector={pattern} tests={skipped}") + if not passed: + failures.append(f"required pass missing: package={package} selector={pattern}") + + if failures: + print("required execution-proof failure:", file=sys.stderr) + for failure in failures: + print(f"- {failure}", file=sys.stderr) + sys.exit(1) + PY + status=$? + fi + exit "$status" - name: Collect G6 failure diagnostics @@ -503,7 +763,7 @@ jobs: export PGPASSWORD="$DB_PASSWORD" collection_ok=1 - while IFS=$'\t' read -r db_name temp_root kind manifest_file; do + while IFS=$'\t' read -r db_name temp_root _kind _manifest_file; do if [ -n "$db_name" ]; then dump_path="${artifact_dir}/db/${db_name}.dump" if pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db_name" --format=custom --no-owner --no-privileges --file "$dump_path"; then @@ -627,13 +887,14 @@ jobs: path: ${{ github.workspace }}/.ci-storage/${{ matrix.codec }} if-no-files-found: ignore - benchmark-matrix: - runs-on: ubuntu-latest + benchmark-integrity: + name: Benchmark integrity (${{ matrix.profile }}) + runs-on: ubuntu-24.04 needs: quality timeout-minutes: 30 services: postgres: - image: postgres:16 + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 env: POSTGRES_USER: coldkeep POSTGRES_PASSWORD: coldkeep @@ -649,7 +910,23 @@ jobs: strategy: fail-fast: false matrix: - compression: [none, zstd] + include: + - profile: none-w1 + compression: none + workers: 1 + dataset: ci-paired-w1-v2 + - profile: none-w4 + compression: none + workers: 4 + dataset: ci-paired-w4-v2 + - profile: zstd-w1 + compression: zstd + workers: 1 + dataset: ci-paired-w1-v2 + - profile: zstd-w4 + compression: zstd + workers: 4 + dataset: ci-paired-w4-v2 steps: - name: Checkout @@ -658,13 +935,13 @@ jobs: - name: Setup Go uses: actions/setup-go@v6 with: - go-version: '1.25.x' + go-version: '1.25.12' cache: true - name: Build CLI - run: go build -o coldkeep ./cmd/coldkeep + run: go build -trimpath -buildvcs=false -o coldkeep ./cmd/coldkeep - - name: Run benchmark (small dataset, workers=1) and capture output + - name: Run hard benchmark integrity validation env: COLDKEEP_DB_AUTO_BOOTSTRAP: true COLDKEEP_CODEC: aes-gcm @@ -679,9 +956,96 @@ jobs: DB_PASSWORD: coldkeep DB_NAME: coldkeep DB_SSLMODE: disable - run: ./coldkeep benchmark run --dataset small --workers 1 --output json | tee benchmark-${{ matrix.compression }}-w1.json + run: | + output_parent="benchmark-integrity-evidence/${{ matrix.profile }}" + output_child="${output_parent}/integrity" + mkdir -p "${output_parent}" + test ! -e "${output_child}" + python3 scripts/benchmark_gate.py integrity \ + --binary ./coldkeep \ + --output-dir "${output_child}" \ + --compression "${{ matrix.compression }}" \ + --workers "${{ matrix.workers }}" \ + --dataset "${{ matrix.dataset }}" \ + --command-timeout-seconds 600 \ + --source-commit "${GITHUB_SHA}" \ + --go-version "$(go version)" \ + --postgres-version "$(psql --version)" \ + --database-image-digest "sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" + + - name: Verify integrity artifact checksums + if: ${{ always() }} + run: | + cd "benchmark-integrity-evidence/${{ matrix.profile }}/integrity" + sha256sum --check checksums.sha256 - - name: Run benchmark (small dataset, workers=4) and capture output + - name: Upload benchmark integrity evidence + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: benchmark-integrity-${{ matrix.profile }} + path: benchmark-integrity-evidence/${{ matrix.profile }}/integrity + if-no-files-found: error + + benchmark-timing-advisory: + name: Benchmark timing advisory (${{ matrix.profile }}) + runs-on: ubuntu-24.04 + needs: quality + timeout-minutes: 20 + services: + postgres: + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 + env: + POSTGRES_USER: coldkeep + POSTGRES_PASSWORD: coldkeep + POSTGRES_DB: coldkeep + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U coldkeep -d coldkeep" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + strategy: + fail-fast: false + matrix: + include: + - profile: none-w1 + compression: none + workers: 1 + mode: uncompressed + baseline: benchmark-baseline-v1.9-packed-aes-gcm-none-small-w1-r1.json + - profile: none-w4 + compression: none + workers: 4 + mode: uncompressed + baseline: benchmark-baseline-v1.9-packed-aes-gcm-none-small-w4-r1.json + - profile: zstd-w1 + compression: zstd + workers: 1 + mode: compressed + baseline: benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w1-r1.json + - profile: zstd-w4 + compression: zstd + workers: 4 + mode: compressed + baseline: benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w4-r1.json + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version: '1.25.12' + cache: true + + - name: Build CLI + run: go build -trimpath -buildvcs=false -o coldkeep ./cmd/coldkeep + + - name: Run historical timing observation env: COLDKEEP_DB_AUTO_BOOTSTRAP: true COLDKEEP_CODEC: aes-gcm @@ -696,51 +1060,66 @@ jobs: DB_PASSWORD: coldkeep DB_NAME: coldkeep DB_SSLMODE: disable - run: ./coldkeep benchmark run --dataset small --workers 4 --output json | tee benchmark-${{ matrix.compression }}-w4.json - - - name: Check regression (workers=1, uncompressed mode) - if: matrix.compression == 'none' - run: >- - python3 scripts/validate_regression_thresholds.py check benchmark-none-w1.json - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-none-small-w1-r1.json - --mode uncompressed - --json-report regression-report-none-w1.json - - - name: Check regression (workers=1, compressed mode) - if: matrix.compression == 'zstd' - run: >- - python3 scripts/validate_regression_thresholds.py check benchmark-zstd-w1.json - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w1-r1.json - --mode compressed - --json-report regression-report-zstd-w1.json - - - name: Check regression (workers=4, uncompressed mode) - if: matrix.compression == 'none' - run: >- - python3 scripts/validate_regression_thresholds.py check benchmark-none-w4.json - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-none-small-w4-r1.json - --mode uncompressed - --json-report regression-report-none-w4.json - - - name: Check regression (workers=4, compressed mode) - if: matrix.compression == 'zstd' - run: >- - python3 scripts/validate_regression_thresholds.py check benchmark-zstd-w4.json - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w4-r1.json - --mode compressed - --json-report regression-report-zstd-w4.json - - - name: Upload benchmark outputs - if: always() + run: | + mkdir -p "benchmark-timing-evidence/${{ matrix.profile }}" + ./coldkeep benchmark run \ + --dataset small \ + --workers "${{ matrix.workers }}" \ + --repeat 1 \ + --output json \ + | tee "benchmark-timing-evidence/${{ matrix.profile }}/benchmark.json" + + - name: Evaluate and verify timing advisory + run: | + evidence_dir="benchmark-timing-evidence/${{ matrix.profile }}" + report="${evidence_dir}/timing-advisory.json" + set +e + python3 scripts/validate_regression_thresholds.py check \ + "${evidence_dir}/benchmark.json" \ + --baseline "benchmarks/v1.9/baselines/${{ matrix.baseline }}" \ + --mode "${{ matrix.mode }}" \ + --policy hosted-advisory \ + --json-report "${report}" + comparator_exit=$? + set -e + [[ -s "${report}" ]] + python3 scripts/validate_regression_thresholds.py verify-advisory-exit \ + --report "${report}" \ + --observed-exit-code "${comparator_exit}" + classification="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["classification"])' "${report}")" + violations="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["violations_count"])' "${report}")" + { + echo "### Benchmark timing advisory: ${{ matrix.profile }}" + echo + echo "Classification: \`${classification}\`" + echo + echo "Historical threshold crossings: ${violations}" + } >> "${GITHUB_STEP_SUMMARY}" + cd "${evidence_dir}" + actual_inventory="$(find . -maxdepth 1 -type f ! -name checksums.sha256 -printf '%f\n' | sort)" + [[ "${actual_inventory}" == $'benchmark.json\ntiming-advisory.json' ]] + sha256sum benchmark.json timing-advisory.json > checksums.sha256 + sha256sum --check checksums.sha256 + case "${comparator_exit}" in + 0|10|11|12) + exit 0 + ;; + 2) + exit 2 + ;; + *) + echo "unexpected benchmark timing advisory exit: ${comparator_exit}" >&2 + exit 2 + ;; + esac + + - name: Upload benchmark timing advisory evidence + if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: benchmark-matrix-${{ matrix.compression }} - path: | - benchmark-${{ matrix.compression }}-w1.json - benchmark-${{ matrix.compression }}-w4.json - regression-report-${{ matrix.compression }}-w1.json - regression-report-${{ matrix.compression }}-w4.json - if-no-files-found: ignore + name: benchmark-timing-advisory-${{ matrix.profile }} + path: benchmark-timing-evidence/${{ matrix.profile }} + if-no-files-found: error critical-coverage-report: name: critical coverage report @@ -791,6 +1170,9 @@ jobs: - name: Show Go environment run: go env + - name: Run native coordination runtime tests + run: go test -v -count=1 -run '^(TestNativeLock|TestWindowsNativeLock|TestProductionCoordinator)' ./internal/coordination + - name: Run path safety cross-platform tests run: go test ./internal/pathsafe/... -run 'TrustedRoot|Symlink|Alias|WritePath' -count=1 @@ -806,7 +1188,7 @@ jobs: ci-required: name: CI Required Gate runs-on: ubuntu-latest - needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-matrix, cross-platform] + needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-integrity, benchmark-timing-advisory, cross-platform] if: ${{ always() }} timeout-minutes: 5 steps: @@ -819,7 +1201,8 @@ jobs: ADVERSARIAL_RESULT: ${{ needs.adversarial.result }} SMOKE_RESULT: ${{ needs.smoke.result }} LEGACY_COMPATIBILITY_RESULT: ${{ needs['legacy-compatibility'].result }} - BENCHMARK_RESULT: ${{ needs['benchmark-matrix'].result }} + BENCHMARK_INTEGRITY_RESULT: ${{ needs['benchmark-integrity'].result }} + BENCHMARK_TIMING_ADVISORY_RESULT: ${{ needs['benchmark-timing-advisory'].result }} CROSS_PLATFORM_RESULT: ${{ needs['cross-platform'].result }} run: | echo "quality=${QUALITY_RESULT}" @@ -829,7 +1212,8 @@ jobs: echo "adversarial=${ADVERSARIAL_RESULT}" echo "smoke=${SMOKE_RESULT}" echo "legacy-compatibility=${LEGACY_COMPATIBILITY_RESULT}" - echo "benchmark=${BENCHMARK_RESULT}" + echo "benchmark-integrity=${BENCHMARK_INTEGRITY_RESULT}" + echo "benchmark-timing-advisory=${BENCHMARK_TIMING_ADVISORY_RESULT}" echo "cross-platform=${CROSS_PLATFORM_RESULT}" if [ "${QUALITY_RESULT}" != "success" ] || \ @@ -839,7 +1223,8 @@ jobs: [ "${ADVERSARIAL_RESULT}" != "success" ] || \ [ "${SMOKE_RESULT}" != "success" ] || \ [ "${LEGACY_COMPATIBILITY_RESULT}" != "success" ] || \ - [ "${BENCHMARK_RESULT}" != "success" ] || \ + [ "${BENCHMARK_INTEGRITY_RESULT}" != "success" ] || \ + [ "${BENCHMARK_TIMING_ADVISORY_RESULT}" != "success" ] || \ [ "${CROSS_PLATFORM_RESULT}" != "success" ]; then echo "One or more required jobs failed or were skipped." exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index c49cfade..5dd7b0bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,125 @@ project, do not start here; start with [README.md](README.md). ------------------------------------------------------------------------ -## v1.13.10 - 2026-07-18 — v1.x Closure Integrity and CI Runtime Hygiene +## v1.13.11 - 2026-08-18 — Safety and Backend Compatibility Gate Closure + +- Completed the Phase 20 pre-release state transition and froze the exact-head + candidate contract. The commit containing that contract must pass + candidate-head CI, Required Gate, CodeQL, and the complete clean local + Profile A gate before one pull request to `main` is authorized. +- Kept benchmark integrity hard-required, timing advisory, and BKC-016 + `Deferred — documented`. Merge, tag, publication, and release-branch + deletion remain separate later operations. + +### Phase 11 benchmark gate diagnostic bootstrap + +- Added a fixed, release-gate-only `ci-stable-v1` calibration fixture, strict + single-envelope schema-v2 reports, per-case database isolation, and an + external sampler with median/MAD statistics and fail-closed evidence checks. +- Added a manual-only, read-only calibration/baseline-capture workflow pinned + to Go 1.25.12, Ubuntu 24.04, and the reviewed PostgreSQL 16 image digest. +- Required CI still uses the historical benchmark gate. No baseline, + threshold, repository runtime, or Phase 12 change is included; Phase 11 + remains blocked on the predetermined calibration. + +### Phase 11 repository coordination contract implemented + +- Added the internal exclusive-only repository coordination contract, stable + error sentinels, non-mutating canonical container-namespace identity, + recovery-safe `.coldkeep-control` namespace, versioned diagnostic owner + metadata, and explicit lease lifecycle helper. +- Added fake-based contract tests for identity aliases, operation policy, + owner metadata, cancellation/deadlines, release errors, nested acquisition, + and independent repositories. Added a pure CLI policy seam that classifies + participating commands without acquiring a lock. +- BKC-016 remains `Deferred — documented`: native Linux/macOS/Windows locking, + CLI acquisition, subprocess contention, crash release, live-GC barriers, and + PostgreSQL advisory session ownership remain Phases 12–13 work. No workflow, + schema, Engine behavior, OS lock, or advisory-lock behavior changed. + +### Phase 10 transaction and row-lock contracts complete + +- Added shared SQLite/PostgreSQL transaction contracts for backend detection, + commit/rollback, read-own-writes, constraint rollback, affected rows, + PostgreSQL `FOR UPDATE`, `NOWAIT`, `SKIP LOCKED`, server-observed blocked-lock + cancellation, and SQLite's intentional clause-omission boundary. +- Added a production-helper integration contract for container NOWAIT + contention/savepoint recovery and deterministic SKIP LOCKED allocation. +- Extended the existing plain-codec internal-package run with + `./internal/container` and five exact PostgreSQL pass-event requirements. + Exact-head CI run `30148670910` at `ad82c959` passed all five events in plain + job `89655223183` and required gate `89656972706`. The initial uncompressed + benchmark variance was resolved by successful same-head rerun `89656813012`; + no benchmark accommodation or production code change was made. + +### Phase 9 engine mutation parity complete + +- Added five shared SQLite/PostgreSQL Engine mutation contracts covering + single-file Store, by-ID and stored-path Remove/Restore, snapshot + create/delete/restore, deterministic failures and partial batches, semantic + repository/container fingerprints, and GC dry-run planning/non-mutation. +- Exact-head CI run `30114444798` at `848e579b` passed quality, all five new + `/postgres` events in plain correctness job `89551564893`, and aggregate + required-gate job `89555865026`. BKC-012/013 are equivalently proven only + for the documented active, uncontended mutation and GC dry-run contracts. +- Extended only the existing plain-codec internal-package JSON event parser and + matching CI audit. No production code, workflow job, package invocation, + codec leg, schema, storage format, or lock behavior changed. + +### Phase 8 snapshot selector determinism closure complete + +- Completed Phase 8 exact-head CI evidence in run `30109561344` at `bcae3576`: + quality, both correctness codecs, required PostgreSQL selector events, + adversarial, stress, long-run, smoke, compatibility, benchmark, + cross-platform, and `CI Required Gate` all passed. BKC-011 is equivalently + proven only for the scoped snapshot list/show/stats/diff selector and + tree-presentation contracts. +- Implemented Phase 8 snapshot selector contracts for deterministic equal-time + list ordering, file-query filtering, invalid direct-engine regex rejection, + pre-cancelled selection, and read non-mutation. CLI diff now preserves + repeated path/prefix query selectors rather than narrowing an unordered map + into one engine path. The direct-engine invalid-regex silent-ignore defect is + corrected by fallible query conversion and propagation through show/diff. + +- Recorded the Phase 0 post-release correction that restored the v1.13.11–v1.13.13 release train. +- Activated executable and reusable release-checklist identity to `1.13.11`. +- Completed the Phase 2 backend compatibility claim matrix, distinguishing + separate evidence from proven parity and recording required-CI gaps. +- Completed the Phase 3 reusable dual-backend test harness with file-backed + SQLite fixtures, optional isolated PostgreSQL scratch databases, strict + cleanup reporting, and catalog-suite adoption. +- Implemented Phase 4 required-CI activation for PostgreSQL-gated internal + package contracts in the plain correctness-matrix codec leg, with JSON + execution-proof enforcement; run `29729981751` confirmed it. Phase 5 + schema/bootstrap/migration parity is Next; no backend parity claim is added. +- Closed Phase 5 schema/bootstrap/migration contract evidence: required + PostgreSQL SCH execution, canonical lint/vet, and selected schema contracts + are recorded without claiming broad schema parity. +- Recorded deterministic G6 shared packed-block corruption reproduction and + added fail-closed protection that refuses partial rebuild of a shared + immutable block while preserving single-member cleanup. +- Recorded final green exact-head CI after one authorized same-SHA retry of a + transient workers=4 uncompressed benchmark anomaly; no benchmark baseline, + workflow, or configuration changed between attempts. +- Completed exact-head catalog contract evidence in CI run `29983479388` at + `db12c3d2`: all six PostgreSQL catalog contract selectors passed, including + CAT-004 after its deterministic `created_at DESC, id DESC` ordering fix, and + the aggregate required gate succeeded. Phase 6 is complete. +- Completed Phase 7 exact-head engine read-side evidence in CI run + `29993172886` at `313d0069`: all four required PostgreSQL engine selectors, + quality, both correctness legs, adversarial validation, and the aggregate + required gate passed. +- Corrected deep verification for a single-connection SQLite handle by fully + materializing and closing eligible-container rows before querying packed + blocks. The bounded regression retains `MaxOpenConns(1)` and the production + packed-storage writer; byte-level verification is unchanged. +- BKC-010 is now equivalently proven for the tested Stats, Inspect, Verify, + context/error, and non-mutation contracts. BKC-011 remains separate evidence + because selector/query behavior is Phase 8 work, which is Next. + +------------------------------------------------------------------------ + +## v1.13.10 - 2026-07-19 — v1.x Closure Integrity and CI Runtime Hygiene - Closed v1.13.9 post-release documentation truth, release-train reconciliation, engine-contract ownership documentation, and the v1.x/v2.0 @@ -40,9 +158,16 @@ project, do not start here; start with [README.md](README.md). database names rather than a cluster-global set. No production behavior changed, and the complete local pre-release gate passed on that remediation commit with no residual benchmark scratch databases. -- The final evidence-restoration commit requires its own clean exact-head gate - before one release pull request is authorized; merge, tag, publication, and - external CI evidence remain pending. +- Public GitHub evidence confirms stable release `Coldkeep v1.13.10 — v1.x + Closure Integrity and CI Runtime Hygiene`, published July 19, 2026 at 18:01; + tag `v1.13.10` targets `423c57815580c39bee4f79ecd81570e9cfa9d273`, the merge + of PR #105. Tag-triggered CI run #502 succeeded with 19 jobs in 18m26s, and + `release/v1.13.10` is absent from the public branch list. The local GitHub + CLI token was invalid; public GitHub pages supplied the independent evidence. +- This remains a valid released closure-integrity and CI-runtime-hygiene + baseline. Its prior final-v1.x conclusion was superseded after release by a + roadmap-to-code audit that found remaining must-before-v2 work; the active + release train is now v1.13.11–v1.13.13. ------------------------------------------------------------------------ diff --git a/PRE_RELEASE_CHECKLIST.md b/PRE_RELEASE_CHECKLIST.md index 6ba5a023..9fa7a5c0 100644 --- a/PRE_RELEASE_CHECKLIST.md +++ b/PRE_RELEASE_CHECKLIST.md @@ -33,8 +33,10 @@ Run: - Section 1: local PostgreSQL and CI-compatible environment setup. - Section 2: quality-equivalent checks. -- Section 3: required CI matrix local equivalents, including smoke, benchmark, - legacy compatibility, and local cross-platform approximation. +- Section 3: required CI matrix local equivalents, including Phase 18 named + backend/storage/recovery/coordination selectors, smoke, hard benchmark + integrity, timing advisory evaluation, legacy compatibility, and the local + cross-platform approximation. - Final local checks: `git diff --check` and `git status -sb`. Profile A is green only when: @@ -42,8 +44,9 @@ Profile A is green only when: - quality checks pass; - required CI matrix local equivalents pass; - smoke passes; -- the benchmark matrix is run and interpreted according to the documented local - variance policy; +- all four benchmark-integrity profiles pass and all four hosted-timing-style + observations produce a valid advisory classification; historical timing + threshold crossings alone do not fail the gate; - `git diff --check` passes; - the working tree is clean or only intentional committed changes remain. @@ -96,7 +99,7 @@ These estimates are broad and environment-dependent: - Quality gate: often several minutes; longer on cold caches or slower CPUs. - Full CI-parity matrix: often tens of minutes because it includes race tests, integration suites, smoke, adversarial tests, and benchmarks. -- Benchmark matrix: sensitive to local CPU scheduling and virtualization, +- Benchmark timing observations: sensitive to local CPU scheduling and virtualization, especially workers=4. - Full release-tag/manual gate: can take substantially longer because it adds manual CLI/operator checks and optional release-specific gates. @@ -283,7 +286,7 @@ python3 scripts/validate_release_state.py --state auto go build -o coldkeep ./cmd/coldkeep -expected_version="1.13.10" +expected_version="1.13.11" human_version=$(./coldkeep version) if [ "$human_version" != "coldkeep version $expected_version" ]; then @@ -311,7 +314,7 @@ fi Expected: local quality checks match CI intent and produce no diff or lint/format failures. -Expected: the built CLI reports exactly 1.13.10 in both human and JSON modes. +Expected: the built CLI reports exactly 1.13.11 in both human and JSON modes. A version mismatch blocks Profile A and release approval. Note: `scripts/clean_test_storage.sh` removes `./storage`, `.ci-storage`, and @@ -363,43 +366,108 @@ done # Reset it before the benchmark block and manual CLI checks in later steps. unset COLDKEEP_CODEC -# benchmark-matrix (CI-equivalent) -# CI always sets COLDKEEP_CODEC=aes-gcm for benchmarks and applies tuned -# lock-retry settings to reduce false-slow results under container contention. -# Both compressions (none, zstd) and both worker counts (1, 4) are required -# CI gates enforced by ci-required — do not skip any combination. -export COLDKEEP_CODEC=aes-gcm -export COLDKEEP_CONTAINER_LOCK_RETRY_ATTEMPTS=12 -export COLDKEEP_CONTAINER_LOCK_RETRY_BASE_WAIT_MS=15 -export COLDKEEP_CONTAINER_LOCK_RETRY_MAX_WAIT_MS=900 +# Phase 18 named execution proof. Every selected test below must report PASS; +# a matching SKIP is a release-gate failure even when `go test` exits zero. +COLDKEEP_CODEC=plain go test -v -race -count=1 ./internal/db \ + -run '^TestMutationRowsAffectedContractAcrossBackends/postgres$' -./coldkeep benchmark run --dataset small --workers 1 --output json | tee benchmark-none-w1.json -./coldkeep benchmark run --dataset small --workers 4 --output json | tee benchmark-none-w4.json +for codec in plain aes-gcm; do + COLDKEEP_CODEC="$codec" go test -v -race -count=1 ./tests/integration/... \ + -run '^TestRoundTripStoreRestore$' +done -COLDKEEP_COMPRESSION=zstd ./coldkeep benchmark run --dataset small --workers 1 --output json | tee benchmark-zstd-w1.json -COLDKEEP_COMPRESSION=zstd ./coldkeep benchmark run --dataset small --workers 4 --output json | tee benchmark-zstd-w4.json +COLDKEEP_CODEC=plain go test -v -race -count=1 ./tests/integration/... \ + -run '^(TestRemoveWithSharedChunksRefCount|TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState)$' -# Regression checks against versioned v1.9 baselines using validate_regression_thresholds.py. -# This mirrors the exact CI gate — all four combinations are required. -python3 scripts/validate_regression_thresholds.py check benchmark-none-w1.json \ - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-none-small-w1-r1.json \ - --mode uncompressed \ - --json-report regression-report-none-w1.json +for codec in plain aes-gcm; do + COLDKEEP_CODEC="$codec" COLDKEEP_LONG_RUN=1 go test -v -race -count=1 \ + ./tests/adversarial/... \ + -run '^(TestAdversarialG6IndependentProcessRepositoryContention|TestAdversarialG6KilledLeaseHolderReleasesRepository|TestAdversarialG6LiveGCExcludesIndependentStoreProcess)$' +done -python3 scripts/validate_regression_thresholds.py check benchmark-none-w4.json \ - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-none-small-w4-r1.json \ - --mode uncompressed \ - --json-report regression-report-none-w4.json +COLDKEEP_CODEC=plain go test -v -race -count=1 ./internal/maintenance \ + -run '^(TestGCAdvisoryLockUsesDedicatedSessionAndReleases|TestRunGCReleasesAdvisoryLockAfterOperationFailure|TestRunGCAdvisoryCleanupFailureReturnsErrorAndDiscardsSession|TestRunGCLiveRefusesSingleConnectionPool)$' -python3 scripts/validate_regression_thresholds.py check benchmark-zstd-w1.json \ - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w1-r1.json \ - --mode compressed \ - --json-report regression-report-zstd-w1.json +# benchmark-integrity and benchmark-timing-advisory (CI-equivalent policy) +# CI always sets COLDKEEP_CODEC=aes-gcm and applies the fixed lock-retry +# settings below. Candidate integrity is hard-required. Historical timing is +# informational when the observation and evaluator are valid; threshold +# crossings must remain visible but do not fail the gate. +export COLDKEEP_CODEC=aes-gcm +export COLDKEEP_CONTAINER_LOCK_RETRY_ATTEMPTS=12 +export COLDKEEP_CONTAINER_LOCK_RETRY_BASE_WAIT_MS=15 +export COLDKEEP_CONTAINER_LOCK_RETRY_MAX_WAIT_MS=900 -python3 scripts/validate_regression_thresholds.py check benchmark-zstd-w4.json \ - --baseline benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w4-r1.json \ - --mode compressed \ - --json-report regression-report-zstd-w4.json +candidate_sha=$(git rev-parse HEAD) +go_version=$(go version) +postgres_version=$(psql --version) +postgres_digest=sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 + +while read -r profile compression workers dataset; do + output_dir="benchmark-integrity-evidence/${profile}/integrity" + test ! -e "$output_dir" + COLDKEEP_COMPRESSION="$compression" \ + python3 scripts/benchmark_gate.py integrity \ + --binary ./coldkeep \ + --output-dir "$output_dir" \ + --compression "$compression" \ + --workers "$workers" \ + --dataset "$dataset" \ + --command-timeout-seconds 600 \ + --source-commit "$candidate_sha" \ + --go-version "$go_version" \ + --postgres-version "$postgres_version" \ + --database-image-digest "$postgres_digest" + (cd "$output_dir" && sha256sum --check checksums.sha256) +done <<'EOF' +none-w1 none 1 ci-paired-w1-v2 +none-w4 none 4 ci-paired-w4-v2 +zstd-w1 zstd 1 ci-paired-w1-v2 +zstd-w4 zstd 4 ci-paired-w4-v2 +EOF + +while read -r profile compression workers mode baseline; do + evidence_dir="benchmark-timing-evidence/${profile}" + mkdir -p "$evidence_dir" + COLDKEEP_COMPRESSION="$compression" \ + ./coldkeep benchmark run \ + --dataset small \ + --workers "$workers" \ + --repeat 1 \ + --output json \ + | tee "${evidence_dir}/benchmark.json" + + set +e + python3 scripts/validate_regression_thresholds.py check \ + "${evidence_dir}/benchmark.json" \ + --baseline "benchmarks/v1.9/baselines/${baseline}" \ + --mode "$mode" \ + --policy hosted-advisory \ + --json-report "${evidence_dir}/timing-advisory.json" + comparator_exit=$? + set -e + + test -s "${evidence_dir}/timing-advisory.json" + python3 scripts/validate_regression_thresholds.py verify-advisory-exit \ + --report "${evidence_dir}/timing-advisory.json" \ + --observed-exit-code "$comparator_exit" + case "$comparator_exit" in + 0|10|11|12) ;; + *) echo "invalid timing-advisory exit: $comparator_exit" >&2; exit 2 ;; + esac + ( + cd "$evidence_dir" + actual_inventory=$(find . -maxdepth 1 -type f ! -name checksums.sha256 -printf '%f\n' | sort) + test "$actual_inventory" = $'benchmark.json\ntiming-advisory.json' + sha256sum benchmark.json timing-advisory.json > checksums.sha256 + sha256sum --check checksums.sha256 + ) +done <<'EOF' +none-w1 none 1 uncompressed benchmark-baseline-v1.9-packed-aes-gcm-none-small-w1-r1.json +none-w4 none 4 uncompressed benchmark-baseline-v1.9-packed-aes-gcm-none-small-w4-r1.json +zstd-w1 zstd 1 compressed benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w1-r1.json +zstd-w4 zstd 4 compressed benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w4-r1.json +EOF unset COLDKEEP_CODEC COLDKEEP_COMPRESSION COLDKEEP_CONTAINER_LOCK_RETRY_ATTEMPTS \ COLDKEEP_CONTAINER_LOCK_RETRY_BASE_WAIT_MS COLDKEEP_CONTAINER_LOCK_RETRY_MAX_WAIT_MS @@ -478,10 +546,10 @@ scripts/smoke.sh Expected: this mirrors the current `ci-required` upstream jobs (`quality`, `correctness-matrix`, `integration-stress`, `integration-long-run`, -`adversarial`, `smoke`, `legacy-compatibility`, and `benchmark-matrix`) across -their documented codec matrices. It also runs a local approximation of the -separate `cross-platform` job, which GitHub Actions must still prove on macOS -and Windows. +`adversarial`, `smoke`, `legacy-compatibility`, `benchmark-integrity`, +`benchmark-timing-advisory`, and `cross-platform`) across their documented +codec/profile matrices. The local cross-platform commands are an approximation; +GitHub Actions must still prove native macOS and Windows runtime. Generate the critical coverage report (mirrors the CI `critical-coverage-report` job; informational, not enforced by `ci-required` in the current workflow, but useful diff --git a/README.md b/README.md index 08fdf7a3..aa9731d3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Coldkeep uses a visual identity based on an ice cube vault: ![CI](https://github.com/franchoy/coldkeep/actions/workflows/ci.yml/badge.svg) ![Go Version](https://img.shields.io/badge/go-1.25+-blue) ![License](https://img.shields.io/badge/license-Apache%202.0-blue) -![Status](https://img.shields.io/badge/status-v1.13.10%20ready%20for%20release%20%2F%20closure%20integrity-blue) +![Status](https://img.shields.io/badge/status-v1.13.11%20ready%20for%20release-blue) ![Release](https://img.shields.io/github/v/release/franchoy/coldkeep?include_prereleases) > Status: v1.9 formalizes transform-based storage semantics (logical/compressed/physical layers) with block-level compression and explicit staged verification, while preserving deterministic restore, GC safety, snapshot semantics, and mixed-repository compatibility. @@ -29,16 +29,20 @@ Coldkeep uses a visual identity based on an ice cube vault: ## Current release state -Coldkeep v1.13.9 completed snapshot create, delete, and restore engine -activation and production CLI routing, then completed the v1.x/v2.0 handoff -review with no mandatory v1.x runtime remediation remaining. Snapshot mutation -routing is no longer deferred. - -v1.13.10 completes the closure-integrity and CI-maintenance train and is ready -for one release pull request. It is not a new runtime feature train. -SQLite-first remains the future local-product direction, PostgreSQL -compatibility remains protected, and v2.0 architectural implementation has not -started. Merge, tag, publication, and external CI evidence are not yet claimed. +v1.13.10 is released: it completed closure-integrity, release-state validation, +CI runtime hygiene, and truthful documentation of known limitations. It remains +a valid released baseline, but its earlier final-v1.x conclusion was superseded +after release by a roadmap-to-code audit that identified remaining +must-before-v2 work. + +v1.13.11 has completed Phases 0–20 and is ready as a pre-release candidate for safety +and backend compatibility gate closure. One pull request is authorized only +after the immutable commit containing the final gate contract passes +candidate-head CI, Required Gate, CodeQL, and the complete clean local Profile +A gate. Merge, tag, and publication remain later operations. v1.13.12 and +v1.13.13 remain required follow-on releases; SQLite-first remains a future +local-product direction, PostgreSQL compatibility remains required, and v2.0 +implementation has not started. coldkeep is a local-first content-addressed storage engine focused on deterministic restore, explicit integrity verification, and safe lifecycle behavior under failure scenarios. diff --git a/VALIDATION_MATRIX.md b/VALIDATION_MATRIX.md index 848798ab..b623edba 100644 --- a/VALIDATION_MATRIX.md +++ b/VALIDATION_MATRIX.md @@ -23,7 +23,12 @@ Guarantee IDs (G1–G17+) are part of the public validation contract. This prevents future "renumbering drift". -All guarantees below are enforced through integration tests and verified under repeated GC / restart / restore cycles. +Guarantees below map to the strongest applicable evidence in the current tree. +That evidence is intentionally heterogeneous: unit, package integration, +adversarial, cross-platform, required hosted CI, manual/local, documentation- +only, and deferred boundaries are distinguished instead of being collapsed +into one `covered` label. A test's existence does not by itself establish that +its execution is required and fail-closed in hosted CI. This document originated from the v0.9/v0.10 trust-validation work and is now the maintained v1.x guarantee-to-evidence contract: v1.0 storage-core @@ -32,13 +37,16 @@ graph coherence guarantees (G10-G13), and v1.3 snapshot-retention guarantees (G1 with v1.4 clarifying lineage semantics, v1.5 adding chunker-evolution compatibility contract clarity, v1.6 adding observability and simulation contract hardening, v1.7 adding controlled-execution performance validation language, v1.8 adding -packed block abstraction and AES-GCM packed-block integration, and v1.9 freezing -transform/verification semantics — +packed block abstraction and AES-GCM packed-block integration, v1.9 freezing +transform/verification semantics, and v1.13.11 adding bounded coordination, +container, decompression, JSON-fidelity, SQL-mutation, and required-CI proof — none of which introduce new guarantee IDs. ## Scope -- Target: single-node trust model for v1.0 core plus v1.1+/v1.2+/v1.3+/v1.4+/v1.5+/v1.6+/v1.7+/v1.8+/v1.9+ interface, observability, and block-abstraction contracts +- Target: single-node trust model for v1.0 core plus maintained v1.x + interface, observability, block-abstraction, same-host coordination, and + integrity-hardening contracts - Surface: existing `verify` and `doctor` contracts (no new top-level validate command) - Goal: each guarantee maps to automated evidence (verify checks, tests, or both) @@ -51,6 +59,8 @@ Reading note: - `Primary verify evidence` names the main runtime verification surface, not every internal helper involved - `Primary test evidence` highlights the most representative automated coverage, not an exhaustive list of all related tests - `covered` means the guarantee is intentionally mapped to concrete automated evidence in the current tree +- `required CI` means hosted workflow policy requires the relevant job or named + pass event; broad job success and named-event proof are recorded separately ## Guarantees to Evidence @@ -61,7 +71,7 @@ Reading note: | G3 | No exposure of partially written or inconsistent data | Recovery + verify model excludes/processes invalid lifecycle states, including standard verify enforcement that each COMPLETED chunk has exactly one blocks row, rollback-safe sealing-marker transitions, quarantine of damaged active containers without harming unrelated live data, ghost-byte sealing-container quarantine with preserved live data, and strict-recovery resynchronization of already-quarantined orphan container size drift instead of surfacing stale metadata as healthy state | `TestStartupRecoverySimulation`, `TestDoctorAbortsProcessingLogicalFilesFromRecoverableState`, `TestVerifyStandard/detects completed chunk missing block row`, `TestStoreSealingMarkerUpdateFailureAbortsSafelyAndRecovers`, `TestStartupRecoveryQuarantinesDamagedActiveContainerAndPreservesOtherLiveData`, `TestStartupRecoveryQuarantinesGhostByteSealingContainerAndPreservesOtherLiveData`, `TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState`, `TestAdversarialG2PreexistingQuarantinedOrphanSizeDriftResyncsAndPreservesHealthyRestore` | covered | | G4 | GC is reference-safe: no reachable chunk is ever deleted | GC liveness checks use `live_ref_count OR pin_count`; verify post-GC integrity | `TestStoreGCRestore`, `TestGCRestorePinRaceContainerNotDeleted`, `TestStoreLifecycleSeededRandomizedOperationOrder` | covered | | G5 | Atomic restore replacement (within single-node local filesystem semantics) | Restore path writes temp + fsync + atomic rename | `TestRestoreFailurePreservesExistingOutput` (explicit atomicity and cleanup), `TestRestoreAtomicityWithTestHook`, `TestRestoreAtomicityWithCorruption`, `TestStoreGCRestore`, `TestSampleDatasetEndToEnd` | covered | -| G6 | Safe in-process concurrent storage operations | Verify catches graph/reference corruption; transactional claims/retries in write path | `TestConcurrentStoreSameFile`, `TestConcurrentStoreSameChunk`, `TestConcurrentStoreFolderStress`, `TestRepeatedJitteredStoreGCRestoreInterleaving`, `TestRepeatedJitteredStoreGCRestoreRemoveInterleaving` (all in-process, shared DB and store path, dedup races, stress) | covered (multi-process contention and external crash overlap not covered; see open work) | +| G6 | Safe in-process concurrent storage operations | Verify catches graph/reference corruption; transactional claims/retries protect in-process writes; the production Coordinator adds an exclusive, fail-fast outer Lease for participating same-host repository operations | In-process: `TestConcurrentStoreSameFile`, `TestConcurrentStoreSameChunk`, `TestConcurrentStoreFolderStress`, `TestRepeatedJitteredStoreGCRestoreInterleaving`, `TestRepeatedJitteredStoreGCRestoreRemoveInterleaving`. Native/Coordinator: `TestNativeLockContentionAndReacquire`, `TestWindowsNativeLockContentionAndReacquire`, `TestProductionCoordinatorIntegratedNativeLifecycle`. Linux process proof: `TestAdversarialG6IndependentProcessRepositoryContention`, `TestAdversarialG6KilledLeaseHolderReleasesRepository`, `TestAdversarialG6LiveGCExcludesIndependentStoreProcess` | covered; required CI names the Linux process events and preserves native Linux/macOS/Windows runtime proof. macOS/Windows subprocess semantics are not separately proven; cross-host and network-filesystem coordination are not claimed | | G7 | Deep corruption detection (payload/offset/tail) | Verify deep validates decoded payload hashes and container continuity, including authenticated AES-GCM decode failures on tampered ciphertext, tampered nonce metadata, wrong-key mismatch, and malformed key configuration (invalid length and invalid encoding) | `TestVerifySystemDeepDetectsChunkDataCorruption`, `TestVerifySystemDeepDetectsAESGCMTamperedCiphertext`, `TestVerifySystemDeepDetectsAESGCMNonceMetadataTampering`, `TestVerifySystemDeepDetectsAESGCMWrongKeyMismatch`, `TestVerifySystemDeepDetectsAESGCMInvalidKeyConfiguration`, `TestVerifySystemDeepDetectsAESGCMInvalidHexKeyConfiguration`, `TestVerifySystemDeepDetectsTrailingBytesAfterLastBlock`, `TestVerifySystemDeepAggregatesChunkErrors` | covered | | G8 | Corrective health gate contract stability | Doctor phase model and JSON/exit-code contract tests | `TestDoctorCommand`, `TestDoctorJSONContractConsistency`, `TestDoctorJSONFailureShortPathSingleMachineReadablePayload`, `TestDoctorRepeatedRecoverableStateConvergesAndPreservesLiveData` | covered | @@ -91,9 +101,9 @@ Use this section for branch-specific additions that are not yet fully covered. | Item | Target evidence | Owner | Status | | --- | --- | --- | --- | | Long-run randomized fault loop expansion | Stress-tier seeded randomized lifecycle loop (`TestStoreLifecycleSeededRandomizedOperationOrder`) plus dedicated long-run soak (`TestRandomizedLongRunLifecycleSoak`) and repeated CI long-run passes | TBD | completed | -| Multi-process contention (non-goal for v1.0 baseline) | Separate post-v1.0 track | TBD | deferred | +| Multi-process contention (non-goal for v1.0 baseline) | Phase 12 native runtime plus Phase 13 Linux independent-process contention, killed-holder release, live-GC exclusion, and PostgreSQL advisory-session evidence | Phases 12–13; named required-CI preservation in Phase 18 | completed within the documented platform boundary | | Atomic restore explicit failure-mode and atomicity | Simulate restore failures before/after rename; verify original output file is preserved, no partial/corrupt final file is visible, and temp files are cleaned up; assert destination file is byte-identical and no temp files remain after failure | `TestRestoreFailurePreservesExistingOutput`, `TestRestoreAtomicityWithTestHook`, `TestRestoreAtomicityWithCorruption`, `TestRestoreFailureDoesNotCorruptDestination` | completed | -| Dry-run support for `remove --stored-path` (deferred beyond v1.2) | Extend remove tx primitive to support rollback-safe preview mode; implement in CLI with `--dry-run` flag; add integration tests validating preview output matches dry-run semantics | Post-v1.2 roadmap | deferred | +| Dry-run support for `remove --stored-path` (deferred beyond v1.2) | Active `Engine.RemoveStoredPaths` dry-run path, production CLI routing, deterministic result reporting, and non-mutation regressions including `TestRemoveStoredPathsDryRunPlansExistingMapping`, `TestRemoveStoredPathsDryRunDoesNotMutateCatalog`, and `TestRemoveStoredPathsDryRunPreservesSnapshotRetentionParityGap` | Completed in v1.13.8 | completed | | Batch delete optimization for remove cascade (v1.4+ optimization) | Current v1.2 implementation uses O(N) per-path delete + invariant check; optimize to batch DELETE + single post-batch invariant check; add micro-benchmarks comparing per-path vs batch semantics; ensure no correctness regression | v1.4 performance enhancement | deferred | | Optional post-batch invariant enforcement strategy | Current v1.2 batch operations preserve invariants per item; future performance-oriented mode may allow post-batch invariant validation while keeping deterministic error semantics | v1.4+ performance track | deferred | | Structured logging for invariant violations (deferred beyond v1.2) | Add optional structured event emission for invariant failures such as `INVARIANT_VIOLATION logical_file_ref_count_mismatch`; cover via CLI/logging contract tests without weakening hard-fail behavior | Post-v1.2 observability track | deferred | @@ -101,8 +111,31 @@ Use this section for branch-specific additions that are not yet fully covered. | Automatic physical-layer repair inside doctor | Keep verify/doctor detect-only for `physical_file` drift even though explicit `repair ref-counts` exists; preserve operator intent and avoid hidden metadata mutation during health checks | Post-v1.2 repair strategy track | deferred | | GC dry-run physical integrity bypass flag | Allow `--force` to skip `CheckPhysicalFileGraphIntegrity` pre-flight for advanced operator scenarios | Future operator tooling sprint | deferred | +## v1.13.11 Hardening and Required Evidence + +These validation groups extend the existing G1-G17 mapping without assigning +new guarantee IDs. They record the difference between automated test coverage +and fail-closed hosted execution proof. + +| Validation group | Contract | Representative evidence | Evidence classification | Hosted requirement and boundary | +| --- | --- | --- | --- | --- | +| Storage and recovery (G1-G5) | Deterministic restore, fail-closed recovery, reference-safe GC, and atomic local replacement | Existing unit/integration/adversarial suites; Phase 18 names `TestRoundTripStoreRestore`, `TestRemoveWithSharedChunksRefCount`, and `TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState` | unit, integration, adversarial, required CI | Selected named events fail closed; broad required jobs remain additional evidence. Single-node/local-filesystem semantics only | +| Coordination (G6) | Same-process protection, supported native primitives, production Coordinator lifecycle, and representative Linux process semantics | Phase 12 native/Coordinator tests; Phase 13 independent-process, killed-holder, live-GC, and advisory-session tests | integration, adversarial, cross-platform, required CI | Native Linux/macOS/Windows plus named Linux process events. No separate macOS/Windows subprocess, cross-host, distributed, or network-filesystem proof | +| Corruption and health (G7-G8) | Deep transform-aware corruption detection and stable doctor/verify contracts | Existing package, integration, and adversarial suites | unit, integration, adversarial, required CI | Broad required jobs execute the suites; not every test has an individual pass-event requirement | +| CLI and physical graph (G9-G13) | Deterministic batch behavior, physical-graph audit, GC refusal, and invariant-aware reporting | Existing command, maintenance, integration, and adversarial suites | unit, integration, adversarial, required CI | Required jobs execute the covered packages; no batch optimization, hidden repair, or bypass behavior is claimed | +| Snapshot retention (G14-G17) | Snapshot roots remain GC-safe and auditable | G14-G17 package/integration tests and explicit adversarial selector | unit, integration, adversarial, required CI | Required adversarial execution retains the G14-G17 family; scope is snapshot-retention correctness only | +| Container integrity (Phase 14) | Validate outer ranges before allocation/I/O; reject header overlap; require header/catalog/physical maximum consistency; preserve persisted maximum; use overflow-safe append; detect short header writes; preserve v0/v1 compatibility | `TestValidateContainerRangeBoundaries`, `TestFileContainerReadAtRejectsInvalidRangeBeforeAllocation`, `TestOpenExistingContainerRejectsHeaderCatalogMaxSizeMismatch`, `TestFileContainerAppendRejectsOverflowAsContainerFull`, `TestStorageBlockReaderUsesCatalogContainerMaxSize` | unit, integration, required CI | Broad required matrix; this is container range/header proof, not decompression proof | +| Bounded decompression (Phase 15) | Enforce exact expected size and the absolute 4 MiB decompression ceiling before decoder/allocation; bound zstd output and decoder memory/window; preserve identity exactness | `TestNoneDecompressRequiresExactExpectedSize`, `TestZstdDecompressRejectsOutputBeyondExpectedSize`, `TestZstdDecompressBoundsConcatenatedFramesAcrossAggregateOutput`, Restore/Verify/reuse bound regressions | unit, integration, adversarial, required CI | Broad required matrix; 4 MiB is a decompression ceiling, not a container maximum | +| JSON integer fidelity (Phase 16) | Preserve exact integer tokens recursively with `UseNumber` and strict EOF for stats, inspect, and simulate-GC | `TestToObjectMapPreservesExactJSONNumbers`, `TestRunStatsCommandJSONPreservesExactLargeIntegers` | unit, integration, required CI | Integers remain JSON numbers and output shape is unchanged; downstream JavaScript precision is not claimed | +| SQL mutation cardinality (Phase 17) | Audit 70 production mutations; harden 20 required-row sites; retain 18 zero-safe and 32 already-safe sites; validate affected rows without imposing blanket exact-one semantics | `TestMutationRowsAffectedContractAcrossBackends`, required container/storage/recovery/repair/remove/GC rollback regressions | unit, dual-backend integration, rollback/adversarial, named required CI | The PostgreSQL cardinality event is required. No claim that every mutation must affect exactly one row | +| CI execution proof (Phase 18) | Reject missing, malformed, skipped, or non-passing selected backend/storage/recovery/coordination events | CI JSON parsers plus `scripts/audit_ci_enforcement.sh` and its regression suite | required CI | Named pass-event proof supplements, rather than replaces, broad job success | +| Benchmarks | Require valid four-profile candidate integrity while treating hosted timing as informational | `benchmark_gate.py integrity`; hosted-advisory comparator and exit verification | required CI integrity, advisory timing, deferred hard performance enforcement | Integrity/evaluator failures block; valid timing threshold crossings do not. Hard timing enforcement is deferred to controlled infrastructure | + ## Exit Criteria 1. Every guarantee row remains mapped to at least one automated test and/or verify check. -2. Quality, correctness-matrix, integration-stress, integration-long-run, legacy-compatibility, benchmark-matrix, and smoke all pass. +2. Quality, correctness-matrix, integration-stress, integration-long-run, + adversarial, legacy-compatibility, smoke, cross-platform, benchmark-integrity, + and benchmark-timing-advisory evaluation all complete successfully; required + named pass events are present with no matching required skip. 3. Contract-sensitive checks (doctor and verify JSON shape, exit codes, failure typing) stay stable. diff --git a/cmd/coldkeep/coordinated_output_spool.go b/cmd/coldkeep/coordinated_output_spool.go new file mode 100644 index 00000000..375ba7c8 --- /dev/null +++ b/cmd/coldkeep/coordinated_output_spool.go @@ -0,0 +1,116 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "sync" +) + +const coordinatedOutputSpoolPattern = "coldkeep-output-*" + +// coordinatedOutputSpool keeps command stdout off heap until the repository +// Lease has been released and the CLI can decide whether the payload is safe +// to emit. It is transient CLI plumbing, not repository state. +type coordinatedOutputSpool struct { + file *os.File + path string + + cleanupOnce sync.Once + cleanupErr error +} + +func newDefaultCoordinatedOutputSpool() (*coordinatedOutputSpool, error) { + return newCoordinatedOutputSpool("") +} + +func newCoordinatedOutputSpool(directory string) (*coordinatedOutputSpool, error) { + file, err := os.CreateTemp(directory, coordinatedOutputSpoolPattern) + if err != nil { + return nil, fmt.Errorf("create coordinated command output spool: %w", err) + } + return &coordinatedOutputSpool{file: file, path: file.Name()}, nil +} + +func (spool *coordinatedOutputSpool) capture(fn func() error) (outputDestination *os.File, err error) { + if spool == nil || spool.file == nil { + return nil, fmt.Errorf("capture coordinated command output: output spool is unavailable") + } + if fn == nil { + return nil, fmt.Errorf("capture coordinated command output: operation is required") + } + + stdoutRedirectMu.Lock() + defer stdoutRedirectMu.Unlock() + + reader, writer, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create coordinated command output pipe: %w", err) + } + outputDestination = os.Stdout + os.Stdout = writer + copyDone := make(chan error, 1) + go func() { + _, copyErr := io.Copy(spool.file, reader) + closeReaderErr := reader.Close() + if copyErr != nil { + copyErr = fmt.Errorf("write coordinated command output spool: %w", copyErr) + } + if closeReaderErr != nil { + closeReaderErr = fmt.Errorf("close coordinated command output pipe reader: %w", closeReaderErr) + } + copyDone <- errors.Join(copyErr, closeReaderErr) + }() + + defer func() { + os.Stdout = outputDestination + closeWriterErr := writer.Close() + copyErr := <-copyDone + if closeWriterErr != nil { + closeWriterErr = fmt.Errorf("close coordinated command output pipe writer: %w", closeWriterErr) + } + err = errors.Join(err, closeWriterErr, copyErr) + }() + + return outputDestination, fn() +} + +func (spool *coordinatedOutputSpool) replayTo(destination io.Writer) error { + if spool == nil || spool.file == nil { + return fmt.Errorf("replay coordinated command output: output spool is unavailable") + } + if destination == nil { + return fmt.Errorf("replay coordinated command output: destination is required") + } + if _, err := spool.file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("seek coordinated command output spool: %w", err) + } + if _, err := io.Copy(destination, spool.file); err != nil { + return fmt.Errorf("replay coordinated command output spool: %w", err) + } + return nil +} + +func (spool *coordinatedOutputSpool) cleanup() error { + if spool == nil { + return nil + } + spool.cleanupOnce.Do(func() { + var closeErr error + if spool.file != nil { + if err := spool.file.Close(); err != nil { + closeErr = fmt.Errorf("close coordinated command output spool: %w", err) + } + } + + var removeErr error + if spool.path != "" { + if err := os.Remove(spool.path); err != nil && !os.IsNotExist(err) { + removeErr = fmt.Errorf("remove coordinated command output spool: %w", err) + } + } + spool.cleanupErr = errors.Join(closeErr, removeErr) + }) + return spool.cleanupErr +} diff --git a/cmd/coldkeep/coordinated_output_spool_test.go b/cmd/coldkeep/coordinated_output_spool_test.go new file mode 100644 index 00000000..6674c479 --- /dev/null +++ b/cmd/coldkeep/coordinated_output_spool_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "crypto/sha256" + "errors" + "fmt" + "hash" + "io" + "os" + "runtime" + "testing" +) + +func TestCoordinatedOutputSpoolStreamsLargePayloadAndCleansUp(t *testing.T) { + const payloadSize = 8 * 1024 * 1024 + const chunkSize = 32 * 1024 + + spoolDirectory := t.TempDir() + spool, err := newCoordinatedOutputSpool(spoolDirectory) + if err != nil { + t.Fatalf("newCoordinatedOutputSpool: %v", err) + } + spoolPath := spool.path + + chunk := make([]byte, chunkSize) + for index := range chunk { + chunk[index] = byte((index*31 + 17) % 251) + } + expectedHash := sha256.New() + _, captureErr := spool.capture(func() error { + remaining := payloadSize + for remaining > 0 { + writeSize := min(remaining, len(chunk)) + written, err := os.Stdout.Write(chunk[:writeSize]) + if err != nil { + return err + } + if written != writeSize { + return io.ErrShortWrite + } + if _, err := expectedHash.Write(chunk[:writeSize]); err != nil { + return err + } + remaining -= writeSize + } + return nil + }) + if captureErr != nil { + t.Fatalf("capture: %v", captureErr) + } + + actualHash := sha256.New() + counter := &countingHashWriter{hash: actualHash} + if err := spool.replayTo(counter); err != nil { + t.Fatalf("replayTo: %v", err) + } + if counter.count != payloadSize { + t.Fatalf("replayed bytes=%d want=%d", counter.count, payloadSize) + } + if got, want := fmt.Sprintf("%x", actualHash.Sum(nil)), fmt.Sprintf("%x", expectedHash.Sum(nil)); got != want { + t.Fatalf("replayed hash=%s want=%s", got, want) + } + + if err := spool.cleanup(); err != nil { + t.Fatalf("cleanup: %v", err) + } + if _, err := os.Lstat(spoolPath); !os.IsNotExist(err) { + t.Fatalf("spool still exists after cleanup, stat err=%v", err) + } + requireDirectoryEmpty(t, spoolDirectory) +} + +func TestCoordinatedOutputSpoolUsesRestrictiveCreateTempPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows temporary-file confidentiality is controlled by inherited ACLs") + } + + spool, err := newCoordinatedOutputSpool(t.TempDir()) + if err != nil { + t.Fatalf("newCoordinatedOutputSpool: %v", err) + } + defer func() { + if err := spool.cleanup(); err != nil { + t.Errorf("cleanup: %v", err) + } + }() + + info, err := spool.file.Stat() + if err != nil { + t.Fatalf("stat spool: %v", err) + } + if permissions := info.Mode().Perm(); permissions&0o077 != 0 { + t.Fatalf("spool permissions=%#o expose group/other access", permissions) + } +} + +func TestCoordinatedOutputSpoolCleanupIsIdempotent(t *testing.T) { + spoolDirectory := t.TempDir() + spool, err := newCoordinatedOutputSpool(spoolDirectory) + if err != nil { + t.Fatalf("newCoordinatedOutputSpool: %v", err) + } + if err := spool.cleanup(); err != nil { + t.Fatalf("first cleanup: %v", err) + } + if err := spool.cleanup(); err != nil { + t.Fatalf("second cleanup: %v", err) + } + requireDirectoryEmpty(t, spoolDirectory) +} + +func TestCoordinatedOutputSpoolReplayFailureIsReturned(t *testing.T) { + spool, err := newCoordinatedOutputSpool(t.TempDir()) + if err != nil { + t.Fatalf("newCoordinatedOutputSpool: %v", err) + } + defer func() { _ = spool.cleanup() }() + + _, err = spool.capture(func() error { + _, writeErr := io.WriteString(os.Stdout, "payload") + return writeErr + }) + if err != nil { + t.Fatalf("capture: %v", err) + } + + wantErr := errors.New("replay failure") + if err := spool.replayTo(failingOutputWriter{err: wantErr}); !errors.Is(err, wantErr) { + t.Fatalf("replay error=%v want errors.Is(%v)", err, wantErr) + } +} + +type countingHashWriter struct { + hash hash.Hash + count int +} + +func (writer *countingHashWriter) Write(data []byte) (int, error) { + written, err := writer.hash.Write(data) + writer.count += written + return written, err +} + +type failingOutputWriter struct { + err error +} + +func (writer failingOutputWriter) Write([]byte) (int, error) { + return 0, writer.err +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 75f0ff31..467829e3 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -3,7 +3,9 @@ package main import ( "bytes" "context" + "crypto/sha256" "database/sql" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -31,9 +33,11 @@ import ( "github.com/franchoy/coldkeep/internal/chunk/simplecdc" clirender "github.com/franchoy/coldkeep/internal/cli/render" "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/coordination" "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/execution" + internalgc "github.com/franchoy/coldkeep/internal/gc" "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/iodebug" "github.com/franchoy/coldkeep/internal/listing" @@ -137,6 +141,17 @@ type parsedCommandLine struct { flags map[string][]string } +type cliRuntime struct { + newCoordinator func() coordination.Coordinator + newOutputSpool func() (*coordinatedOutputSpool, error) + resolveIdentity func(string) (coordination.Identity, error) + newOwner func(coordination.Operation, coordination.Identity, string, time.Time) (coordination.Owner, error) + recover func(cliOutputMode) (recovery.Report, error) + dispatch func(parsedCommandLine, cliOutputMode) error + renderSuccess func(parsedCommandLine, cliOutputMode) + now func() time.Time +} + type verifyOutputSummary struct { BlocksChecked int64 PhysicalHashChecked int64 @@ -435,57 +450,10 @@ var snapshotStatsPhase = func(ctx context.Context, db *sql.DB, id string) (*snap var deleteSnapshotPhase = snapshot.DeleteSnapshot var snapshotDeleteLineagePreviewPhase = loadSnapshotDeleteLineagePreview var diffSnapshotsPhase = func(ctx context.Context, db *sql.DB, baseID, targetID string, query *snapshot.SnapshotQuery) (*snapshot.SnapshotDiffResult, error) { - // Engine.SnapshotDiff is active, but this CLI workflow adapts richer query - // semantics into the narrower engine request surface. Only one exact path - // and one prefix can cross this seam. Active method presence does not prove - // complete read-side workflow ownership; early v2.0 owns the remaining - // ownership decision. - eng, err := engine.New(engine.Config{DB: db}) - if err != nil { - return nil, err - } - req := engine.SnapshotDiffRequest{BaseID: baseID, TargetID: targetID} - if query != nil { - eq := engine.SnapshotQuery{ - Pattern: query.Pattern, - MinSize: query.MinSize, - MaxSize: query.MaxSize, - ModifiedAfter: query.ModifiedAfter, - ModifiedBefore: query.ModifiedBefore, - } - for p := range query.ExactPaths { - eq.Path = p - break // engine.SnapshotQuery.Path is a single exact path - } - if len(query.Prefixes) > 0 { - eq.Prefix = query.Prefixes[0] - } - if query.Regex != nil { - eq.Regex = query.Regex.String() - } - req.Query = eq - } - result, err := eng.SnapshotDiff(ctx, req) - if err != nil { - return nil, err - } - entries := make([]snapshot.SnapshotDiffEntry, len(result.Entries)) - for i, e := range result.Entries { - entries[i] = snapshot.SnapshotDiffEntry{ - Path: e.StoredPath, - Type: snapshot.DiffType(e.Change), - } - } - return &snapshot.SnapshotDiffResult{ - BaseSnapshotID: result.BaseID, - TargetSnapshotID: result.TargetID, - Entries: entries, - Summary: snapshot.SnapshotDiffSummary{ - Added: int64(result.Summary.Added), - Removed: int64(result.Summary.Removed), - Modified: int64(result.Summary.Modified), - }, - }, nil + // The CLI accepts repeated path and prefix selectors. Keep that full + // snapshot-domain query shape instead of narrowing it through the current + // single-path engine seam. + return snapshot.DiffSnapshots(ctx, db, baseID, targetID, query) } var diffSnapshotSummaryPhase = func(ctx context.Context, db *sql.DB, baseID, targetID string) (*snapshot.SnapshotDiffSummary, error) { eng, err := engine.New(engine.Config{DB: db}) @@ -695,6 +663,19 @@ func main() { } func runCLI(args []string) int { + return runCLIWithRuntime(args, cliRuntime{ + newCoordinator: coordination.NewCoordinator, + newOutputSpool: newDefaultCoordinatedOutputSpool, + resolveIdentity: coordination.ResolveIdentity, + newOwner: coordination.NewOwner, + recover: runStartupRecoveryWithOptionalLogBuffering, + dispatch: dispatchCLICommand, + renderSuccess: printCLISuccess, + now: time.Now, + }) +} + +func runCLIWithRuntime(args []string, runtime cliRuntime) int { startupMode := inferOutputModeFromArgs(args) if startupMode == outputModeJSON { prevOutput := log.Writer() @@ -712,18 +693,6 @@ func runCLI(args []string) int { return exitSuccess } - if shouldRunStartupRecovery(args) { - recoveryReport, recoveryErr := runStartupRecoveryWithOptionalLogBuffering(startupMode) - if recoveryErr != nil { - log.Printf("System recovery failed: %v\n", recoveryErr) - } - emitStartupRecoveryReport(startupMode, recoveryReport, recoveryErr) - - if startupMode != outputModeJSON { - checkEnvFilePermissions() - } - } - parsed, err := parseCommandLine(args, flagsWithValues) if err != nil { return printCLIError(err, startupMode) @@ -743,6 +712,91 @@ func runCLI(args []string) int { return printCLIError(err, startupMode) } + policy := repositoryCoordinationPolicyFor(parsed) + err = executeCLICommand(args, parsed, outputMode, policy, runtime) + if err != nil { + return printCLIError(err, outputMode) + } + + runtime.renderSuccess(parsed, outputMode) + + return exitSuccess +} + +func executeCLICommand( + args []string, + parsed parsedCommandLine, + outputMode cliOutputMode, + policy repositoryCoordinationPolicy, + runtime cliRuntime, +) (err error) { + if !policy.Required { + return runtime.dispatch(parsed, outputMode) + } + newOutputSpool := runtime.newOutputSpool + if newOutputSpool == nil { + newOutputSpool = newDefaultCoordinatedOutputSpool + } + outputSpool, err := newOutputSpool() + if err != nil { + return err + } + if outputSpool == nil { + return fmt.Errorf("create coordinated command output spool: factory returned nil spool") + } + defer func() { + err = errors.Join(err, outputSpool.cleanup()) + }() + + identity, err := runtime.resolveIdentity(container.ContainersDir) + if err != nil { + return err + } + owner, err := runtime.newOwner(policy.Operation, identity, version.String(), runtime.now()) + if err != nil { + return err + } + request := coordination.Request{ + Operation: policy.Operation, + Mode: policy.Mode, + Owner: owner, + } + + var operationErr error + var outputDestination *os.File + lifecycleErr := coordination.WithLease( + context.Background(), + cliRepositoryCoordinator{delegate: runtime.newCoordinator()}, + identity, + request, + func() error { + outputDestination, operationErr = outputSpool.capture(func() error { + if shouldRunStartupRecovery(args) { + recoveryReport, recoveryErr := runtime.recover(outputMode) + if recoveryErr != nil { + log.Printf("System recovery failed: %v\n", recoveryErr) + } + emitStartupRecoveryReport(outputMode, recoveryReport, recoveryErr) + if outputMode != outputModeJSON { + checkEnvFilePermissions() + } + } + return runtime.dispatch(parsed, outputMode) + }) + return operationErr + }, + ) + shouldReplayOutput := lifecycleErr == nil || operationErr != nil + if shouldReplayOutput && outputDestination != nil { + if replayErr := outputSpool.replayTo(outputDestination); replayErr != nil { + return errors.Join(lifecycleErr, replayErr) + } + } + return lifecycleErr +} + +func dispatchCLICommand(parsed parsedCommandLine, outputMode cliOutputMode) error { + var err error switch parsed.method { case "init": err = initCommand(parsed, outputMode) @@ -793,14 +847,7 @@ func runCLI(args []string) int { default: err = usageErrorf("unknown command: %s", parsed.method) } - - if err != nil { - return printCLIError(err, outputMode) - } - - printCLISuccess(parsed, outputMode) - - return exitSuccess + return err } func runStartupRecoveryWithOptionalLogBuffering(mode cliOutputMode) (recovery.Report, error) { @@ -914,6 +961,7 @@ func emitStartupRecoveryReport(mode cliOutputMode, report recovery.Report, err e } func printCLIError(err error, mode cliOutputMode) int { + err = stableCLIError(err) code := classifyExitCode(err) message := strings.TrimSpace(err.Error()) publicCode := publicErrorCode(err, code) @@ -1032,7 +1080,7 @@ func printCLISuccess(parsed parsedCommandLine, mode cliOutputMode) { // These commands emit their own structured JSON payload. // Keep this list in sync with TestPrintCLISuccessJSONCommandPolicy. switch parsed.method { - case "store", "store-folder", "restore", "remove", "repair", "gc", "list", "search", "stats", "inspect", "simulate", "doctor", "snapshot", "config", "version", "-v", "--version", "verify": + case "store", "store-folder", "restore", "remove", "repair", "gc", "list", "search", "stats", "inspect", "simulate", "benchmark", "doctor", "snapshot", "config", "version", "-v", "--version", "verify": return } @@ -3226,12 +3274,14 @@ func (p *perfTimer) Spans() []perfSpan { return p.spans } // BenchmarkRunReport is the output payload for `coldkeep benchmark run`. type BenchmarkRunReport struct { - GeneratedAtUTC string `json:"generated_at_utc"` - Dataset string `json:"dataset"` - Repeat int `json:"repeat"` - Execution BenchmarkExecution `json:"execution"` - ExecutionStats BenchmarkExecutionStats `json:"execution_stats"` - Rows []BenchmarkRunCaseRow `json:"rows"` + SchemaVersion int `json:"schema_version"` + GeneratedAtUTC string `json:"generated_at_utc"` + Dataset string `json:"dataset"` + Repeat int `json:"repeat"` + Fixture corebenchmark.FixtureDescriptor `json:"fixture"` + Execution BenchmarkExecution `json:"execution"` + ExecutionStats BenchmarkExecutionStats `json:"execution_stats"` + Rows []BenchmarkRunCaseRow `json:"rows"` } // BenchmarkExecution captures execution policy knobs used for this run. @@ -3263,11 +3313,12 @@ type BenchmarkIOMetrics struct { // BenchmarkRunCaseRow is one per-case benchmark summary row. type BenchmarkRunCaseRow struct { - Case string `json:"case"` - DurationMs int64 `json:"duration_ms"` - ThroughputMBps float64 `json:"throughput_mbps"` - Execution BenchmarkExecution `json:"execution"` - ExecutionStats BenchmarkExecutionStats `json:"execution_stats"` + Case string `json:"case"` + DurationMs int64 `json:"duration_ms"` + ThroughputMBps float64 `json:"throughput_mbps"` + Execution BenchmarkExecution `json:"execution"` + ExecutionStats BenchmarkExecutionStats `json:"execution_stats"` + DiagnosticFinalState json.RawMessage `json:"diagnostic_final_state,omitempty"` } func runBenchmarkCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { @@ -3368,6 +3419,9 @@ func runBenchmarkRunCommand(parsed parsedCommandLine, outputMode cliOutputMode) return usageErrorf("invalid --repeat value %q (must be integer > 0)", rawRepeat) } } + if corebenchmark.RequiresCaseDatabaseIsolation(preset) && repeat != 1 { + return usageErrorf("%s requires --repeat 1; independent sampling is owned by the benchmark gate scripts", preset) + } report, err := runCoreBenchmarkPhase(preset, repeat, opts) if err != nil { @@ -3444,12 +3498,20 @@ func compareWithBaseline(current BenchmarkRunReport, baselinePath string, thresh // The baseline file is the full JSON envelope written by --output json. var envelope struct { - Data BenchmarkRunReport `json:"data"` + Status string `json:"status"` + Command string `json:"command"` + Data BenchmarkRunReport `json:"data"` } if err := json.Unmarshal(raw, &envelope); err != nil { return fmt.Errorf("parse baseline %q: %w", baselinePath, err) } + if envelope.Status != "ok" || envelope.Command != "benchmark" { + return fmt.Errorf("parse baseline %q: expected successful benchmark envelope", baselinePath) + } baseline := envelope.Data + if err := validateLegacyBenchmarkComparisonInputs(baseline, current); err != nil { + return fmt.Errorf("validate baseline %q: %w", baselinePath, err) + } baselineByCase := make(map[string]BenchmarkRunCaseRow, len(baseline.Rows)) for _, row := range baseline.Rows { @@ -3466,10 +3528,7 @@ func compareWithBaseline(current BenchmarkRunReport, baselinePath string, thresh var regressions []regressionEntry for _, row := range current.Rows { - base, ok := baselineByCase[row.Case] - if !ok { - continue // new case added since baseline was captured; skip - } + base := baselineByCase[row.Case] if base.DurationMs > 0 { delta := float64(row.DurationMs-base.DurationMs) / float64(base.DurationMs) * 100.0 if delta > thresholdPct { @@ -3511,20 +3570,85 @@ func compareWithBaseline(current BenchmarkRunReport, baselinePath string, thresh return fmt.Errorf("benchmark regression: %d case(s) exceeded the %.0f%% degradation threshold", len(regressions), thresholdPct) } -func runCoreBenchmark(preset corebenchmark.DatasetPreset, repeat int, opts execution.Options) (BenchmarkRunReport, error) { - if err := runBenchmarkDeterminismPhase(preset, opts); err != nil { - return BenchmarkRunReport{}, err +func validateLegacyBenchmarkComparisonInputs(baseline, current BenchmarkRunReport) error { + if len(baseline.Rows) == 0 || len(current.Rows) == 0 { + return fmt.Errorf("benchmark reports must contain at least one case") + } + if baseline.Dataset != "" && current.Dataset != "" && baseline.Dataset != current.Dataset { + return fmt.Errorf("dataset mismatch: baseline=%q current=%q", baseline.Dataset, current.Dataset) + } + if baseline.Repeat > 0 && current.Repeat > 0 && baseline.Repeat != current.Repeat { + return fmt.Errorf("repeat mismatch: baseline=%d current=%d", baseline.Repeat, current.Repeat) + } + if baseline.Execution.StoreFolderWorkers > 0 && + current.Execution.StoreFolderWorkers > 0 && + baseline.Execution != current.Execution { + return fmt.Errorf("execution policy mismatch") + } + + validateRows := func(label string, rows []BenchmarkRunCaseRow) (map[string]struct{}, error) { + seen := make(map[string]struct{}, len(rows)) + for index, row := range rows { + if strings.TrimSpace(row.Case) == "" { + return nil, fmt.Errorf("%s case at index %d has empty name", label, index) + } + if _, exists := seen[row.Case]; exists { + return nil, fmt.Errorf("%s contains duplicate case %q", label, row.Case) + } + if row.DurationMs <= 0 { + return nil, fmt.Errorf("%s case %q has non-positive duration", label, row.Case) + } + if math.IsNaN(row.ThroughputMBps) || math.IsInf(row.ThroughputMBps, 0) || row.ThroughputMBps <= 0 { + return nil, fmt.Errorf("%s case %q has invalid throughput", label, row.Case) + } + seen[row.Case] = struct{}{} + } + return seen, nil } - report, err := runPresetInTemporaryDatabase(preset, repeat, opts, "report") + baselineCases, err := validateRows("baseline", baseline.Rows) + if err != nil { + return err + } + currentCases, err := validateRows("current", current.Rows) + if err != nil { + return err + } + if len(baselineCases) != len(currentCases) { + return fmt.Errorf("case set mismatch") + } + for index, row := range baseline.Rows { + if _, ok := currentCases[row.Case]; !ok { + return fmt.Errorf("current report is missing case %q", row.Case) + } + if current.Rows[index].Case != row.Case { + return fmt.Errorf("case order mismatch at index %d", index) + } + } + return nil +} + +func runCoreBenchmark(preset corebenchmark.DatasetPreset, repeat int, opts execution.Options) (BenchmarkRunReport, error) { + var report corebenchmark.RunReport + var err error + if corebenchmark.RequiresCaseDatabaseIsolation(preset) { + report, err = runGatePresetWithIsolatedDatabases(preset, repeat, opts) + } else { + if err := runBenchmarkDeterminismPhase(preset, opts); err != nil { + return BenchmarkRunReport{}, err + } + report, err = runPresetInTemporaryDatabase(preset, repeat, opts, "report") + } if err != nil { return BenchmarkRunReport{}, err } out := BenchmarkRunReport{ + SchemaVersion: 2, GeneratedAtUTC: report.GeneratedAtUTC, Dataset: string(report.Dataset), Repeat: report.Repeat, + Fixture: report.Fixture, Execution: BenchmarkExecution{ StoreFolderWorkers: opts.StoreFolderWorkers, PipelineDepth: opts.PipelineDepth, @@ -3534,11 +3658,12 @@ func runCoreBenchmark(preset corebenchmark.DatasetPreset, repeat int, opts execu } type runAgg struct { - durationMs int64 - bytes int64 - files int - execution BenchmarkExecution - stats BenchmarkExecutionStats + durationMs int64 + bytes int64 + files int + execution BenchmarkExecution + stats BenchmarkExecutionStats + diagnosticFinalState json.RawMessage } caseAgg := make(map[string]runAgg) caseOrder := make([]string, 0) @@ -3553,6 +3678,9 @@ func runCoreBenchmark(preset corebenchmark.DatasetPreset, repeat int, opts execu Deterministic: result.Execution.Deterministic, } agg.stats.WorkersUsed = result.ExecStats.WorkersUsed + agg.diagnosticFinalState = append(json.RawMessage(nil), result.DiagnosticFinalState...) + } else if !bytes.Equal(agg.diagnosticFinalState, result.DiagnosticFinalState) { + return BenchmarkRunReport{}, fmt.Errorf("diagnostic final state changed across repeats for case %q", result.Name) } agg.durationMs += result.Metrics.Duration.Milliseconds() agg.bytes += result.Metrics.BytesProcessed @@ -3581,11 +3709,12 @@ func runCoreBenchmark(preset corebenchmark.DatasetPreset, repeat int, opts execu throughput = (float64(agg.bytes) / (1024.0 * 1024.0)) / seconds } out.Rows = append(out.Rows, BenchmarkRunCaseRow{ - Case: caseName, - DurationMs: agg.durationMs, - ThroughputMBps: throughput, - Execution: agg.execution, - ExecutionStats: agg.stats, + Case: caseName, + DurationMs: agg.durationMs, + ThroughputMBps: throughput, + Execution: agg.execution, + ExecutionStats: agg.stats, + DiagnosticFinalState: agg.diagnosticFinalState, }) out.ExecutionStats.TotalFiles += agg.stats.TotalFiles out.ExecutionStats.TotalBytes += agg.stats.TotalBytes @@ -3607,6 +3736,918 @@ func runCoreBenchmark(preset corebenchmark.DatasetPreset, repeat int, opts execu return out, nil } +func runGatePresetWithIsolatedDatabases( + preset corebenchmark.DatasetPreset, + repeat int, + opts execution.Options, +) (corebenchmark.RunReport, error) { + if repeat <= 0 { + return corebenchmark.RunReport{}, fmt.Errorf("repeat must be > 0") + } + cfg, err := corebenchmark.PresetScenarioConfig(preset) + if err != nil { + return corebenchmark.RunReport{}, err + } + cfg.ColdkeepExecutable = resolveSelfExecutable() + cfg.Codec = strings.TrimSpace(os.Getenv("COLDKEEP_CODEC")) + cfg.Compression = strings.TrimSpace(os.Getenv("COLDKEEP_COMPRESSION")) + cfg.Execution = opts + cfg.CaseEnvironmentFactory = func(caseName string) (map[string]string, func() error, error) { + dbName, cleanup, err := createTemporaryBenchmarkDatabase("gate-" + caseName) + if err != nil { + return nil, nil, err + } + return map[string]string{ + "DB_NAME": dbName, + "COLDKEEP_DB_AUTO_BOOTSTRAP": "true", + "COLDKEEP_STORE_FOLDER_WORKERS": strconv.Itoa(opts.StoreFolderWorkers), + }, cleanup, nil + } + report := corebenchmark.RunReport{ + GeneratedAtUTC: time.Now().UTC().Format(time.RFC3339), + Dataset: preset, + Repeat: repeat, + Fixture: corebenchmark.FixtureDescriptorFor(preset, cfg), + Iterations: make([]corebenchmark.IterationReport, 0, repeat), + } + for iteration := 1; iteration <= repeat; iteration++ { + iterCfg := cfg + iterCfg.RunTag = fmt.Sprintf("iter-%02d", iteration) + results, err := corebenchmark.RunBenchmarkWithEnvironmentFactoryAndObserver( + corebenchmark.CoreScenarios(iterCfg), + iterCfg.CaseEnvironmentFactory, + captureBenchmarkDiagnosticFinalState, + ) + report.Iterations = append(report.Iterations, corebenchmark.IterationReport{ + Iteration: iteration, + Results: results, + }) + if err != nil { + return report, err + } + } + return report, nil +} + +const benchmarkDiagnosticFinalStateSchemaVersion = 2 + +type benchmarkDiagnosticDigest struct { + Count int64 `json:"count"` + TotalBytes int64 `json:"total_bytes"` + SHA256 string `json:"sha256"` +} + +type benchmarkDiagnosticStatusTotals struct { + Completed int64 `json:"completed"` + Processing int64 `json:"processing"` + Aborted int64 `json:"aborted"` +} + +type benchmarkDiagnosticGC struct { + TotalChunks int64 `json:"total_chunks"` + ReachableChunks int64 `json:"reachable_chunks"` + UnreachableChunks int64 `json:"unreachable_chunks"` + LogicallyReclaimableBytes int64 `json:"logically_reclaimable_bytes"` + PhysicallyReclaimableBytes int64 `json:"physically_reclaimable_bytes"` + PackedBlocksLive int64 `json:"packed_blocks_live"` + PackedBlocksDead int64 `json:"packed_blocks_dead"` + PackedBytesLive int64 `json:"packed_bytes_live"` + PackedBytesReclaimable int64 `json:"packed_bytes_reclaimable"` + RetainedDeadBytes int64 `json:"retained_dead_bytes"` +} + +type benchmarkDiagnosticVerification struct { + BlocksChecked int64 `json:"blocks_checked"` + PhysicalHashesChecked int64 `json:"physical_hashes_checked"` + CompressedHashesChecked int64 `json:"compressed_hashes_checked"` + LogicalHashesChecked int64 `json:"logical_hashes_checked"` + CompressedBlocksChecked int64 `json:"compressed_blocks_checked"` + PhysicalFileIssues int64 `json:"physical_file_issues"` + SnapshotMembershipRows int64 `json:"snapshot_membership_rows"` + SnapshotReachabilityIssues int64 `json:"snapshot_reachability_issues"` +} + +type benchmarkDiagnosticPhysical struct { + ContainerCount int64 `json:"container_count"` + StorageBlockCount int64 `json:"storage_block_count"` + LegacyBlockCount int64 `json:"legacy_block_count"` + ChunkReferenceCount int64 `json:"chunk_reference_count"` + PayloadBytes int64 `json:"payload_bytes"` + ContainerBytes int64 `json:"container_bytes"` + CanonicalSHA256 string `json:"canonical_sha256"` +} + +type benchmarkDiagnosticFinalState struct { + SchemaVersion int `json:"schema_version"` + ActiveLogicalNamespace benchmarkDiagnosticDigest `json:"active_logical_namespace"` + LogicalCatalog benchmarkDiagnosticDigest `json:"logical_catalog"` + LogicalStatuses benchmarkDiagnosticStatusTotals `json:"logical_statuses"` + ChunkGraph benchmarkDiagnosticDigest `json:"chunk_graph"` + RestoredTree benchmarkDiagnosticDigest `json:"restored_tree"` + Snapshots benchmarkDiagnosticDigest `json:"snapshots"` + SnapshotCount int64 `json:"snapshot_count"` + GC benchmarkDiagnosticGC `json:"gc"` + Verification benchmarkDiagnosticVerification `json:"verification"` + Physical benchmarkDiagnosticPhysical `json:"physical"` + PhysicalLayoutSHA256 string `json:"physical_layout_sha256"` +} + +type benchmarkActiveLogicalRawRow struct { + ID int64 + Path string + FileHash string + TotalSize int64 + Status string + ChunkerVersion string +} + +type benchmarkActiveLogicalCanonicalRow struct { + Path string `json:"path"` + FileHash string `json:"file_hash"` + TotalSize int64 `json:"total_size"` + Status string `json:"status"` + ChunkerVersion string `json:"chunker_version"` +} + +type benchmarkLogicalCatalogRawRow struct { + ID int64 + FileHash string + TotalSize int64 + Status string + RefCount int64 + ChunkerVersion string + ActivePathCount int64 + SnapshotReferenceCount int64 +} + +type benchmarkLogicalCatalogCanonicalRow struct { + FileHash string `json:"file_hash"` + TotalSize int64 `json:"total_size"` + Status string `json:"status"` + RefCount int64 `json:"ref_count"` + ChunkerVersion string `json:"chunker_version"` + ActivePathCount int64 `json:"active_path_count"` + SnapshotReferenceCount int64 `json:"snapshot_reference_count"` + ReachabilityClass string `json:"reachability_class"` +} + +type benchmarkChunkGraphRawRow struct { + LogicalID int64 + ChunkID int64 + FileHash string + FileSize int64 + ChunkOrder int64 + ChunkHash string + ChunkSize int64 + Status string + LiveRefCount int64 + PinCount int64 + ChunkerVersion string +} + +type benchmarkChunkGraphCanonicalRow struct { + FileHash string `json:"file_hash"` + FileSize int64 `json:"file_size"` + ChunkOrder int64 `json:"chunk_order"` + ChunkHash string `json:"chunk_hash"` + ChunkSize int64 `json:"chunk_size"` + Status string `json:"status"` + LiveRefCount int64 `json:"live_ref_count"` + PinCount int64 `json:"pin_count"` + ChunkerVersion string `json:"chunker_version"` +} + +type benchmarkSnapshotRawRow struct { + SnapshotID string + Type string + Label string + ParentID string + Path string + FileHash string + Size int64 +} + +type benchmarkSnapshotCanonicalRow struct { + Type string `json:"type"` + Label string `json:"label"` + HasParent bool `json:"has_parent"` + Path string `json:"path"` + FileHash string `json:"file_hash"` + Size int64 `json:"size"` +} + +type benchmarkPhysicalRawRow struct { + BlockID int64 + ContainerID int64 + FormatVersion int + Codec string + PlaintextSize int64 + CompressionCodec string + CompressionLevel sql.NullInt64 + CompressedSize sql.NullInt64 + StoredSize int64 + BlockHash string + CompressedHash string + PhysicalHash string + ContainerOffset int64 + ChunkHash string + OffsetInBlock sql.NullInt64 + SizeInBlock sql.NullInt64 +} + +type benchmarkPhysicalCanonicalRow struct { + FormatVersion int `json:"format_version"` + Codec string `json:"codec"` + CompressionCodec string `json:"compression_codec"` + CompressionLevel *int64 `json:"compression_level"` + ChunkHash string `json:"chunk_hash"` + ChunkSize *int64 `json:"chunk_size"` + UnreferencedBlockHash string `json:"unreferenced_block_hash,omitempty"` +} + +type benchmarkPhysicalLayoutRow struct { + Canonical benchmarkPhysicalCanonicalRow `json:"canonical"` + PlaintextSize int64 `json:"plaintext_size"` + CompressedSize *int64 `json:"compressed_size"` + StoredSize int64 `json:"stored_size"` + BlockHash string `json:"block_hash"` + CompressedHash string `json:"compressed_hash"` + PhysicalHash string `json:"physical_hash"` + ContainerOffset int64 `json:"container_offset"` + OffsetInBlock *int64 `json:"offset_in_block"` +} + +type benchmarkContainerRawRow struct { + ID int64 + Sealed bool + Sealing bool + Quarantine bool + CurrentSize int64 + MaxSize int64 + ContainerHash string +} + +type benchmarkContainerLayout struct { + Sealed bool `json:"sealed"` + Sealing bool `json:"sealing"` + Quarantine bool `json:"quarantine"` + CurrentSize int64 `json:"current_size"` + MaxSize int64 `json:"max_size"` + ContainerHash string `json:"container_hash"` + Rows []benchmarkPhysicalLayoutRow `json:"rows"` +} + +func captureBenchmarkDiagnosticFinalState(_ string, benchmarkContext corebenchmark.BenchmarkContext) (json.RawMessage, error) { + dbName := strings.TrimSpace(benchmarkContext.ExtraEnv["DB_NAME"]) + if dbName == "" { + return nil, fmt.Errorf("diagnostic observer requires an isolated database") + } + connStr, err := db.BuildPostgresConnStringFromEnv(dbName) + if err != nil { + return nil, fmt.Errorf("build diagnostic database connection: %w", err) + } + dbconn, err := sql.Open("postgres", connStr) + if err != nil { + return nil, fmt.Errorf("open diagnostic database: %w", err) + } + defer func() { _ = dbconn.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + if err := dbconn.PingContext(ctx); err != nil { + return nil, fmt.Errorf("ping diagnostic database: %w", err) + } + + state, err := buildBenchmarkDiagnosticFinalState(ctx, dbconn, benchmarkContext) + if err != nil { + return nil, err + } + encoded, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("encode diagnostic final state: %w", err) + } + return encoded, nil +} + +func benchmarkCanonicalDigest(value any) (string, error) { + encoded, err := json.Marshal(value) + if err != nil { + return "", err + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]), nil +} + +func canonicalBenchmarkPath(root, rawPath string) (string, error) { + if strings.TrimSpace(rawPath) == "" { + return "", nil + } + absRoot, err := filepath.Abs(root) + if err != nil { + return "", fmt.Errorf("resolve benchmark data root: %w", err) + } + absPath, err := filepath.Abs(rawPath) + if err != nil { + return "", fmt.Errorf("resolve benchmark path: %w", err) + } + rel, err := filepath.Rel(absRoot, absPath) + if err != nil { + return "", fmt.Errorf("relativize benchmark path: %w", err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return "", fmt.Errorf("benchmark path is outside the isolated data root") + } + return filepath.ToSlash(rel), nil +} + +func canonicalBenchmarkSnapshotPath(dataRoot, rawPath string) (string, error) { + if strings.TrimSpace(rawPath) == "" { + return "", nil + } + if filepath.IsAbs(rawPath) { + return canonicalBenchmarkPath(dataRoot, rawPath) + } + normalizedRaw := filepath.ToSlash(filepath.Clean(filepath.FromSlash(rawPath))) + absDataRoot, err := filepath.Abs(dataRoot) + if err != nil { + return "", fmt.Errorf("resolve benchmark data root: %w", err) + } + normalizedRoot := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(absDataRoot)), "/") + if normalizedRaw == normalizedRoot { + return ".", nil + } + if strings.HasPrefix(normalizedRaw, normalizedRoot+"/") { + return strings.TrimPrefix(normalizedRaw, normalizedRoot+"/"), nil + } + cleaned := filepath.Clean(filepath.FromSlash(normalizedRaw)) + if cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) || filepath.IsAbs(cleaned) { + return "", fmt.Errorf("snapshot path escapes the isolated data root") + } + return filepath.ToSlash(cleaned), nil +} + +func canonicalizeBenchmarkActiveLogicalRows(rows []benchmarkActiveLogicalRawRow, dataRoot string) ([]benchmarkActiveLogicalCanonicalRow, error) { + out := make([]benchmarkActiveLogicalCanonicalRow, 0, len(rows)) + seenPaths := make(map[string]struct{}, len(rows)) + for _, row := range rows { + pathValue, err := canonicalBenchmarkPath(dataRoot, row.Path) + if err != nil { + return nil, err + } + if pathValue == "" { + return nil, fmt.Errorf("active logical path is empty") + } + if _, exists := seenPaths[pathValue]; exists { + return nil, fmt.Errorf("duplicate canonical active logical path") + } + seenPaths[pathValue] = struct{}{} + out = append(out, benchmarkActiveLogicalCanonicalRow{ + Path: pathValue, FileHash: row.FileHash, TotalSize: row.TotalSize, + Status: row.Status, ChunkerVersion: row.ChunkerVersion, + }) + } + sort.Slice(out, func(i, j int) bool { + left, _ := json.Marshal(out[i]) + right, _ := json.Marshal(out[j]) + return bytes.Compare(left, right) < 0 + }) + return out, nil +} + +func benchmarkLogicalReachabilityClass(activePathCount, snapshotReferenceCount int64) string { + switch { + case activePathCount > 0 && snapshotReferenceCount > 0: + return "shared" + case activePathCount > 0: + return "current_only" + case snapshotReferenceCount > 0: + return "snapshot_only" + default: + return "unreachable_history" + } +} + +func canonicalizeBenchmarkLogicalCatalogRows(rows []benchmarkLogicalCatalogRawRow) []benchmarkLogicalCatalogCanonicalRow { + out := make([]benchmarkLogicalCatalogCanonicalRow, 0, len(rows)) + for _, row := range rows { + out = append(out, benchmarkLogicalCatalogCanonicalRow{ + FileHash: row.FileHash, TotalSize: row.TotalSize, Status: row.Status, + RefCount: row.RefCount, ChunkerVersion: row.ChunkerVersion, + ActivePathCount: row.ActivePathCount, SnapshotReferenceCount: row.SnapshotReferenceCount, + ReachabilityClass: benchmarkLogicalReachabilityClass(row.ActivePathCount, row.SnapshotReferenceCount), + }) + } + sort.Slice(out, func(i, j int) bool { + left, _ := json.Marshal(out[i]) + right, _ := json.Marshal(out[j]) + return bytes.Compare(left, right) < 0 + }) + return out +} + +func canonicalizeBenchmarkChunkGraphRows(rows []benchmarkChunkGraphRawRow) []benchmarkChunkGraphCanonicalRow { + out := make([]benchmarkChunkGraphCanonicalRow, 0, len(rows)) + for _, row := range rows { + out = append(out, benchmarkChunkGraphCanonicalRow{ + FileHash: row.FileHash, FileSize: row.FileSize, ChunkOrder: row.ChunkOrder, + ChunkHash: row.ChunkHash, ChunkSize: row.ChunkSize, Status: row.Status, + LiveRefCount: row.LiveRefCount, PinCount: row.PinCount, ChunkerVersion: row.ChunkerVersion, + }) + } + sort.Slice(out, func(i, j int) bool { + left, _ := json.Marshal(out[i]) + right, _ := json.Marshal(out[j]) + return bytes.Compare(left, right) < 0 + }) + return out +} + +func nullableInt64Pointer(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + v := value.Int64 + return &v +} + +func canonicalizeBenchmarkPhysicalRows(rows []benchmarkPhysicalRawRow) []benchmarkPhysicalCanonicalRow { + out := make([]benchmarkPhysicalCanonicalRow, 0, len(rows)) + for _, row := range rows { + unreferencedBlockHash := "" + if row.ChunkHash == "" { + unreferencedBlockHash = row.BlockHash + } + out = append(out, benchmarkPhysicalCanonicalRow{ + FormatVersion: row.FormatVersion, Codec: row.Codec, + CompressionCodec: row.CompressionCodec, CompressionLevel: nullableInt64Pointer(row.CompressionLevel), + ChunkHash: row.ChunkHash, ChunkSize: nullableInt64Pointer(row.SizeInBlock), + UnreferencedBlockHash: unreferencedBlockHash, + }) + } + sort.Slice(out, func(i, j int) bool { + left, _ := json.Marshal(out[i]) + right, _ := json.Marshal(out[j]) + return bytes.Compare(left, right) < 0 + }) + return out +} + +func buildBenchmarkDiagnosticFinalState( + ctx context.Context, + dbconn *sql.DB, + benchmarkContext corebenchmark.BenchmarkContext, +) (benchmarkDiagnosticFinalState, error) { + activeRaw, activeBytes, err := readBenchmarkActiveLogicalNamespace(ctx, dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, err + } + activeRows, err := canonicalizeBenchmarkActiveLogicalRows(activeRaw, benchmarkContext.DataPath) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("canonicalize active logical namespace: %w", err) + } + activeDigest, err := benchmarkCanonicalDigest(activeRows) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("digest active logical namespace: %w", err) + } + + catalogRaw, catalogBytes, statuses, err := readBenchmarkLogicalCatalog(ctx, dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, err + } + catalogRows := canonicalizeBenchmarkLogicalCatalogRows(catalogRaw) + catalogDigest, err := benchmarkCanonicalDigest(catalogRows) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("digest logical catalog: %w", err) + } + + chunkRaw, chunkBytes, err := readBenchmarkChunkGraph(ctx, dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, err + } + chunkRows := canonicalizeBenchmarkChunkGraphRows(chunkRaw) + chunkDigest, err := benchmarkCanonicalDigest(chunkRows) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("digest chunk graph: %w", err) + } + + restoredRows, restoredBytes, err := readBenchmarkRestoredTree(filepath.Join(benchmarkContext.RepoPath, "restore-output")) + if err != nil { + return benchmarkDiagnosticFinalState{}, err + } + restoredDigest, err := benchmarkCanonicalDigest(restoredRows) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("digest restored tree: %w", err) + } + + snapshotRows, snapshotCount, snapshotBytes, err := readBenchmarkSnapshotState(ctx, dbconn, benchmarkContext.DataPath) + if err != nil { + return benchmarkDiagnosticFinalState{}, err + } + snapshotDigest, err := benchmarkCanonicalDigest(snapshotRows) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("digest snapshot membership: %w", err) + } + + gcPlan, err := internalgc.BuildPlan(ctx, dbconn, internalgc.PlanOptions{}) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("capture GC reachability totals: %w", err) + } + verifyTotals, err := countVerifySummaryForSystem(dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("capture verification totals: %w", err) + } + physicalAudit, err := verify.CheckPhysicalFileGraphIntegrity(dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("capture physical-file verification totals: %w", err) + } + snapshotAudit, err := verify.CheckSnapshotReachabilityIntegrity(dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, fmt.Errorf("capture snapshot verification totals: %w", err) + } + + physical, layoutDigest, err := readBenchmarkPhysicalState(ctx, dbconn) + if err != nil { + return benchmarkDiagnosticFinalState{}, err + } + + return benchmarkDiagnosticFinalState{ + SchemaVersion: benchmarkDiagnosticFinalStateSchemaVersion, + ActiveLogicalNamespace: benchmarkDiagnosticDigest{Count: int64(len(activeRows)), TotalBytes: activeBytes, SHA256: activeDigest}, + LogicalCatalog: benchmarkDiagnosticDigest{Count: int64(len(catalogRows)), TotalBytes: catalogBytes, SHA256: catalogDigest}, + LogicalStatuses: statuses, + ChunkGraph: benchmarkDiagnosticDigest{Count: int64(len(chunkRows)), TotalBytes: chunkBytes, SHA256: chunkDigest}, + RestoredTree: benchmarkDiagnosticDigest{Count: int64(len(restoredRows)), TotalBytes: restoredBytes, SHA256: restoredDigest}, + Snapshots: benchmarkDiagnosticDigest{Count: int64(len(snapshotRows)), TotalBytes: snapshotBytes, SHA256: snapshotDigest}, + SnapshotCount: snapshotCount, + GC: benchmarkDiagnosticGC{ + TotalChunks: gcPlan.TotalChunks, ReachableChunks: gcPlan.ReachableChunks, + UnreachableChunks: gcPlan.UnreachableChunks, LogicallyReclaimableBytes: gcPlan.ReclaimableBytes, + PhysicallyReclaimableBytes: gcPlan.PhysicallyReclaimableBytes, + PackedBlocksLive: gcPlan.Summary.PackedBlocksLive, PackedBlocksDead: gcPlan.Summary.PackedBlocksDead, + PackedBytesLive: gcPlan.Summary.PackedBytesLive, PackedBytesReclaimable: gcPlan.Summary.PackedBytesReclaimable, + RetainedDeadBytes: gcPlan.Summary.RetainedDeadBytesDueToPackedBlocks, + }, + Verification: benchmarkDiagnosticVerification{ + BlocksChecked: verifyTotals.BlocksChecked, PhysicalHashesChecked: verifyTotals.PhysicalHashChecked, + CompressedHashesChecked: verifyTotals.CompressedHashChecked, LogicalHashesChecked: verifyTotals.LogicalHashChecked, + CompressedBlocksChecked: verifyTotals.CompressedBlocksChecked, + PhysicalFileIssues: physicalAudit.OrphanPhysicalFileRows + physicalAudit.LogicalRefCountMismatches + physicalAudit.NegativeLogicalRefCounts, + SnapshotMembershipRows: snapshotAudit.SnapshotFileRows, + SnapshotReachabilityIssues: snapshotAudit.OrphanSnapshotPathRefs + snapshotAudit.DuplicateSnapshotPathPairs + + snapshotAudit.OrphanSnapshotLogicalRefs + snapshotAudit.InvalidSnapshotLifecycleStates + snapshotAudit.RetainedMissingChunkGraph, + }, + Physical: physical, + PhysicalLayoutSHA256: layoutDigest, + }, nil +} + +func readBenchmarkActiveLogicalNamespace( + ctx context.Context, + dbconn *sql.DB, +) ([]benchmarkActiveLogicalRawRow, int64, error) { + rows, err := dbconn.QueryContext(ctx, ` + SELECT lf.id, pf.path, lf.file_hash, lf.total_size, lf.status, lf.chunker_version + FROM physical_file pf + JOIN logical_file lf ON lf.id = pf.logical_file_id + `) + if err != nil { + return nil, 0, fmt.Errorf("query diagnostic active logical namespace: %w", err) + } + defer func() { _ = rows.Close() }() + var out []benchmarkActiveLogicalRawRow + var totalBytes int64 + for rows.Next() { + var row benchmarkActiveLogicalRawRow + if err := rows.Scan(&row.ID, &row.Path, &row.FileHash, &row.TotalSize, &row.Status, &row.ChunkerVersion); err != nil { + return nil, 0, fmt.Errorf("scan diagnostic active logical namespace: %w", err) + } + out = append(out, row) + totalBytes += row.TotalSize + } + if err := rows.Err(); err != nil { + return nil, 0, fmt.Errorf("iterate diagnostic active logical namespace: %w", err) + } + return out, totalBytes, nil +} + +func readBenchmarkLogicalCatalog( + ctx context.Context, + dbconn *sql.DB, +) ([]benchmarkLogicalCatalogRawRow, int64, benchmarkDiagnosticStatusTotals, error) { + rows, err := dbconn.QueryContext(ctx, ` + WITH active_paths AS ( + SELECT logical_file_id, COUNT(*) AS active_path_count + FROM physical_file + GROUP BY logical_file_id + ), snapshot_references AS ( + SELECT logical_file_id, COUNT(*) AS snapshot_reference_count + FROM snapshot_file + GROUP BY logical_file_id + ) + SELECT lf.id, lf.file_hash, lf.total_size, lf.status, lf.ref_count, lf.chunker_version, + COALESCE(ap.active_path_count, 0), COALESCE(sr.snapshot_reference_count, 0) + FROM logical_file lf + LEFT JOIN active_paths ap ON ap.logical_file_id = lf.id + LEFT JOIN snapshot_references sr ON sr.logical_file_id = lf.id + `) + if err != nil { + return nil, 0, benchmarkDiagnosticStatusTotals{}, fmt.Errorf("query diagnostic logical catalog: %w", err) + } + defer func() { _ = rows.Close() }() + var out []benchmarkLogicalCatalogRawRow + var totalBytes int64 + var statuses benchmarkDiagnosticStatusTotals + for rows.Next() { + var row benchmarkLogicalCatalogRawRow + if err := rows.Scan( + &row.ID, &row.FileHash, &row.TotalSize, &row.Status, &row.RefCount, &row.ChunkerVersion, + &row.ActivePathCount, &row.SnapshotReferenceCount, + ); err != nil { + return nil, 0, benchmarkDiagnosticStatusTotals{}, fmt.Errorf("scan diagnostic logical catalog: %w", err) + } + out = append(out, row) + totalBytes += row.TotalSize + switch row.Status { + case "COMPLETED": + statuses.Completed++ + case "PROCESSING": + statuses.Processing++ + case "ABORTED": + statuses.Aborted++ + } + } + if err := rows.Err(); err != nil { + return nil, 0, benchmarkDiagnosticStatusTotals{}, fmt.Errorf("iterate diagnostic logical catalog: %w", err) + } + return out, totalBytes, statuses, nil +} + +func readBenchmarkChunkGraph(ctx context.Context, dbconn *sql.DB) ([]benchmarkChunkGraphRawRow, int64, error) { + rows, err := dbconn.QueryContext(ctx, ` + SELECT lf.id, c.id, lf.file_hash, lf.total_size, fc.chunk_order, + c.chunk_hash, c.size, c.status, c.live_ref_count, c.pin_count, c.chunker_version + FROM file_chunk fc + JOIN logical_file lf ON lf.id = fc.logical_file_id + JOIN chunk c ON c.id = fc.chunk_id + `) + if err != nil { + return nil, 0, fmt.Errorf("query diagnostic chunk graph: %w", err) + } + defer func() { _ = rows.Close() }() + var out []benchmarkChunkGraphRawRow + var totalBytes int64 + for rows.Next() { + var row benchmarkChunkGraphRawRow + if err := rows.Scan(&row.LogicalID, &row.ChunkID, &row.FileHash, &row.FileSize, &row.ChunkOrder, + &row.ChunkHash, &row.ChunkSize, &row.Status, &row.LiveRefCount, &row.PinCount, &row.ChunkerVersion); err != nil { + return nil, 0, fmt.Errorf("scan diagnostic chunk graph: %w", err) + } + out = append(out, row) + totalBytes += row.ChunkSize + } + if err := rows.Err(); err != nil { + return nil, 0, fmt.Errorf("iterate diagnostic chunk graph: %w", err) + } + return out, totalBytes, nil +} + +type benchmarkRestoredCanonicalRow struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +func readBenchmarkRestoredTree(root string) ([]benchmarkRestoredCanonicalRow, int64, error) { + if _, err := os.Stat(root); err != nil { + if os.IsNotExist(err) { + return make([]benchmarkRestoredCanonicalRow, 0), 0, nil + } + return nil, 0, fmt.Errorf("inspect restored tree: %w", err) + } + hashes, err := corebenchmark.HashRestoredTree(root) + if err != nil { + return nil, 0, fmt.Errorf("capture restored tree: %w", err) + } + paths := make([]string, 0, len(hashes)) + for relativePath := range hashes { + paths = append(paths, relativePath) + } + sort.Strings(paths) + out := make([]benchmarkRestoredCanonicalRow, 0, len(paths)) + var totalBytes int64 + for _, relativePath := range paths { + info, err := os.Stat(filepath.Join(root, filepath.FromSlash(relativePath))) + if err != nil { + return nil, 0, fmt.Errorf("stat restored tree file: %w", err) + } + out = append(out, benchmarkRestoredCanonicalRow{Path: relativePath, SHA256: hashes[relativePath], Size: info.Size()}) + totalBytes += info.Size() + } + return out, totalBytes, nil +} + +func readBenchmarkSnapshotState( + ctx context.Context, + dbconn *sql.DB, + dataRoot string, +) ([]benchmarkSnapshotCanonicalRow, int64, int64, error) { + rows, err := dbconn.QueryContext(ctx, ` + SELECT s.id, s.type, COALESCE(s.label, ''), COALESCE(s.parent_id, ''), + COALESCE(sp.path, ''), COALESCE(lf.file_hash, ''), COALESCE(sf.size, 0) + FROM snapshot s + LEFT JOIN snapshot_file sf ON sf.snapshot_id = s.id + LEFT JOIN snapshot_path sp ON sp.id = sf.path_id + LEFT JOIN logical_file lf ON lf.id = sf.logical_file_id + `) + if err != nil { + return nil, 0, 0, fmt.Errorf("query diagnostic snapshot membership: %w", err) + } + defer func() { _ = rows.Close() }() + var rawRows []benchmarkSnapshotRawRow + snapshotIDs := make(map[string]struct{}) + for rows.Next() { + var row benchmarkSnapshotRawRow + if err := rows.Scan(&row.SnapshotID, &row.Type, &row.Label, &row.ParentID, &row.Path, &row.FileHash, &row.Size); err != nil { + return nil, 0, 0, fmt.Errorf("scan diagnostic snapshot membership: %w", err) + } + rawRows = append(rawRows, row) + snapshotIDs[row.SnapshotID] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, 0, 0, fmt.Errorf("iterate diagnostic snapshot membership: %w", err) + } + out := make([]benchmarkSnapshotCanonicalRow, 0, len(rawRows)) + var totalBytes int64 + for _, row := range rawRows { + pathValue, err := canonicalBenchmarkSnapshotPath(dataRoot, row.Path) + if err != nil { + return nil, 0, 0, fmt.Errorf("canonicalize snapshot path: %w", err) + } + out = append(out, benchmarkSnapshotCanonicalRow{ + Type: row.Type, Label: row.Label, HasParent: row.ParentID != "", + Path: pathValue, FileHash: row.FileHash, Size: row.Size, + }) + totalBytes += row.Size + } + sort.Slice(out, func(i, j int) bool { + left, _ := json.Marshal(out[i]) + right, _ := json.Marshal(out[j]) + return bytes.Compare(left, right) < 0 + }) + return out, int64(len(snapshotIDs)), totalBytes, nil +} + +func readBenchmarkPhysicalState( + ctx context.Context, + dbconn *sql.DB, +) (benchmarkDiagnosticPhysical, string, error) { + storageRows, err := dbconn.QueryContext(ctx, ` + SELECT sb.id, sb.container_id, sb.format_version, sb.codec, sb.plaintext_size, + sb.compression_codec, sb.compression_level, sb.compressed_size, sb.stored_size, + encode(sb.block_hash, 'hex'), COALESCE(encode(sb.compressed_hash, 'hex'), ''), + COALESCE(encode(sb.physical_hash, 'hex'), ''), sb.container_offset, + COALESCE(ch.chunk_hash, ''), cbr.offset_in_block, cbr.size_in_block + FROM storage_blocks sb + LEFT JOIN chunk_block_refs cbr ON cbr.block_id = sb.id + LEFT JOIN chunk ch ON ch.id = cbr.chunk_id + `) + if err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("query diagnostic storage blocks: %w", err) + } + var rawRows []benchmarkPhysicalRawRow + for storageRows.Next() { + var row benchmarkPhysicalRawRow + if err := storageRows.Scan( + &row.BlockID, &row.ContainerID, &row.FormatVersion, &row.Codec, &row.PlaintextSize, + &row.CompressionCodec, &row.CompressionLevel, &row.CompressedSize, &row.StoredSize, + &row.BlockHash, &row.CompressedHash, &row.PhysicalHash, &row.ContainerOffset, + &row.ChunkHash, &row.OffsetInBlock, &row.SizeInBlock, + ); err != nil { + _ = storageRows.Close() + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("scan diagnostic storage block: %w", err) + } + rawRows = append(rawRows, row) + } + if err := storageRows.Close(); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("close diagnostic storage block rows: %w", err) + } + if err := storageRows.Err(); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("iterate diagnostic storage blocks: %w", err) + } + + legacyRows, err := dbconn.QueryContext(ctx, ` + SELECT b.id, b.container_id, b.format_version, b.codec, b.plaintext_size, + b.stored_size, b.block_offset, c.chunk_hash + FROM blocks b + JOIN chunk c ON c.id = b.chunk_id + `) + if err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("query diagnostic legacy blocks: %w", err) + } + for legacyRows.Next() { + var row benchmarkPhysicalRawRow + if err := legacyRows.Scan( + &row.BlockID, &row.ContainerID, &row.FormatVersion, &row.Codec, + &row.PlaintextSize, &row.StoredSize, &row.ContainerOffset, &row.ChunkHash, + ); err != nil { + _ = legacyRows.Close() + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("scan diagnostic legacy block: %w", err) + } + row.CompressionCodec = "none" + row.BlockHash = row.ChunkHash + row.OffsetInBlock = sql.NullInt64{Int64: 0, Valid: true} + row.SizeInBlock = sql.NullInt64{Int64: row.PlaintextSize, Valid: true} + rawRows = append(rawRows, row) + } + if err := legacyRows.Close(); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("close diagnostic legacy block rows: %w", err) + } + if err := legacyRows.Err(); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("iterate diagnostic legacy blocks: %w", err) + } + + var physical benchmarkDiagnosticPhysical + if err := dbconn.QueryRowContext(ctx, ` + SELECT + (SELECT COUNT(*) FROM container), + (SELECT COUNT(*) FROM storage_blocks), + (SELECT COUNT(*) FROM blocks), + (SELECT COUNT(*) FROM chunk_block_refs), + COALESCE((SELECT SUM(stored_size) FROM storage_blocks), 0) + + COALESCE((SELECT SUM(stored_size) FROM blocks), 0), + COALESCE((SELECT SUM(current_size) FROM container), 0) + `).Scan( + &physical.ContainerCount, &physical.StorageBlockCount, &physical.LegacyBlockCount, + &physical.ChunkReferenceCount, &physical.PayloadBytes, &physical.ContainerBytes, + ); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("query diagnostic physical totals: %w", err) + } + canonicalRows := canonicalizeBenchmarkPhysicalRows(rawRows) + physical.CanonicalSHA256, err = benchmarkCanonicalDigest(canonicalRows) + if err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("digest canonical physical content: %w", err) + } + + containerRows, err := dbconn.QueryContext(ctx, ` + SELECT id, sealed, sealing, quarantine, current_size, max_size, COALESCE(container_hash, '') + FROM container + `) + if err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("query diagnostic containers: %w", err) + } + containers := make(map[int64]*benchmarkContainerLayout) + for containerRows.Next() { + var row benchmarkContainerRawRow + if err := containerRows.Scan(&row.ID, &row.Sealed, &row.Sealing, &row.Quarantine, &row.CurrentSize, &row.MaxSize, &row.ContainerHash); err != nil { + _ = containerRows.Close() + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("scan diagnostic container: %w", err) + } + containers[row.ID] = &benchmarkContainerLayout{ + Sealed: row.Sealed, Sealing: row.Sealing, Quarantine: row.Quarantine, + CurrentSize: row.CurrentSize, MaxSize: row.MaxSize, ContainerHash: row.ContainerHash, + Rows: make([]benchmarkPhysicalLayoutRow, 0), + } + } + if err := containerRows.Close(); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("close diagnostic container rows: %w", err) + } + if err := containerRows.Err(); err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("iterate diagnostic containers: %w", err) + } + for _, raw := range rawRows { + containerLayout, ok := containers[raw.ContainerID] + if !ok { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("diagnostic block references a missing container") + } + canonical := canonicalizeBenchmarkPhysicalRows([]benchmarkPhysicalRawRow{raw})[0] + containerLayout.Rows = append(containerLayout.Rows, benchmarkPhysicalLayoutRow{ + Canonical: canonical, PlaintextSize: raw.PlaintextSize, + CompressedSize: nullableInt64Pointer(raw.CompressedSize), StoredSize: raw.StoredSize, + BlockHash: raw.BlockHash, CompressedHash: raw.CompressedHash, PhysicalHash: raw.PhysicalHash, + ContainerOffset: raw.ContainerOffset, OffsetInBlock: nullableInt64Pointer(raw.OffsetInBlock), + }) + } + layouts := make([]benchmarkContainerLayout, 0, len(containers)) + for _, layout := range containers { + sort.Slice(layout.Rows, func(i, j int) bool { + left, _ := json.Marshal(layout.Rows[i]) + right, _ := json.Marshal(layout.Rows[j]) + return bytes.Compare(left, right) < 0 + }) + layouts = append(layouts, *layout) + } + sort.Slice(layouts, func(i, j int) bool { + left, _ := json.Marshal(layouts[i]) + right, _ := json.Marshal(layouts[j]) + return bytes.Compare(left, right) < 0 + }) + layoutDigest, err := benchmarkCanonicalDigest(layouts) + if err != nil { + return benchmarkDiagnosticPhysical{}, "", fmt.Errorf("digest physical layout: %w", err) + } + return physical, layoutDigest, nil +} + type benchmarkStateSnapshot struct { ChunkCount int64 LogicalFileHashes []string @@ -4614,6 +5655,9 @@ func renderSnapshotTreeLines(items []snapshot.Snapshot) []string { } emitTopLevel := func(node snapshot.Snapshot) { + if _, seen := visited[node.ID]; seen { + return + } if len(lines) > 0 { lines = append(lines, "") } diff --git a/cmd/coldkeep/main_test.go b/cmd/coldkeep/main_test.go index ad0c1d89..5102ca5a 100644 --- a/cmd/coldkeep/main_test.go +++ b/cmd/coldkeep/main_test.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "log" + "math" "os" "path/filepath" "reflect" @@ -37,6 +38,370 @@ import ( type singleChunkV2CLITestChunker struct{} +func TestBenchmarkDiagnosticFingerprintsIgnoreRowOrderAndGeneratedIDs(t *testing.T) { + dataRoot := t.TempDir() + activeA := []benchmarkActiveLogicalRawRow{ + {ID: 1001, Path: filepath.Join(dataRoot, "b.txt"), FileHash: "hash-b", TotalSize: 2, Status: "COMPLETED", ChunkerVersion: "v2-fastcdc"}, + {ID: 1000, Path: filepath.Join(dataRoot, "a.txt"), FileHash: "hash-a", TotalSize: 1, Status: "COMPLETED", ChunkerVersion: "v2-fastcdc"}, + } + activeB := []benchmarkActiveLogicalRawRow{ + {ID: 9000, Path: filepath.Join(dataRoot, "a.txt"), FileHash: "hash-a", TotalSize: 1, Status: "COMPLETED", ChunkerVersion: "v2-fastcdc"}, + {ID: 9001, Path: filepath.Join(dataRoot, "b.txt"), FileHash: "hash-b", TotalSize: 2, Status: "COMPLETED", ChunkerVersion: "v2-fastcdc"}, + } + canonicalA, err := canonicalizeBenchmarkActiveLogicalRows(activeA, dataRoot) + if err != nil { + t.Fatalf("canonicalize active logical rows A: %v", err) + } + canonicalB, err := canonicalizeBenchmarkActiveLogicalRows(activeB, dataRoot) + if err != nil { + t.Fatalf("canonicalize active logical rows B: %v", err) + } + digestA, _ := benchmarkCanonicalDigest(canonicalA) + digestB, _ := benchmarkCanonicalDigest(canonicalB) + if digestA != digestB { + t.Fatalf("active namespace digest changed with row order/IDs: %s != %s", digestA, digestB) + } + + catalogA := []benchmarkLogicalCatalogRawRow{ + {ID: 101, FileHash: "hash-b", TotalSize: 2, Status: "COMPLETED", RefCount: 0, ChunkerVersion: "v2-fastcdc", SnapshotReferenceCount: 1}, + {ID: 100, FileHash: "hash-a", TotalSize: 1, Status: "COMPLETED", RefCount: 1, ChunkerVersion: "v2-fastcdc", ActivePathCount: 1}, + } + catalogB := []benchmarkLogicalCatalogRawRow{ + {ID: 900, FileHash: "hash-a", TotalSize: 1, Status: "COMPLETED", RefCount: 1, ChunkerVersion: "v2-fastcdc", ActivePathCount: 1}, + {ID: 901, FileHash: "hash-b", TotalSize: 2, Status: "COMPLETED", RefCount: 0, ChunkerVersion: "v2-fastcdc", SnapshotReferenceCount: 1}, + } + catalogDigestA, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkLogicalCatalogRows(catalogA)) + catalogDigestB, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkLogicalCatalogRows(catalogB)) + if catalogDigestA != catalogDigestB { + t.Fatalf("logical catalog digest changed with row order/IDs: %s != %s", catalogDigestA, catalogDigestB) + } + + graphA := []benchmarkChunkGraphRawRow{ + {LogicalID: 1, ChunkID: 20, FileHash: "file", FileSize: 3, ChunkOrder: 1, ChunkHash: "chunk-b", ChunkSize: 2, Status: "COMPLETED", LiveRefCount: 1, ChunkerVersion: "v2-fastcdc"}, + {LogicalID: 1, ChunkID: 10, FileHash: "file", FileSize: 3, ChunkOrder: 0, ChunkHash: "chunk-a", ChunkSize: 1, Status: "COMPLETED", LiveRefCount: 1, ChunkerVersion: "v2-fastcdc"}, + } + graphB := []benchmarkChunkGraphRawRow{ + {LogicalID: 800, ChunkID: 700, FileHash: "file", FileSize: 3, ChunkOrder: 0, ChunkHash: "chunk-a", ChunkSize: 1, Status: "COMPLETED", LiveRefCount: 1, ChunkerVersion: "v2-fastcdc"}, + {LogicalID: 800, ChunkID: 701, FileHash: "file", FileSize: 3, ChunkOrder: 1, ChunkHash: "chunk-b", ChunkSize: 2, Status: "COMPLETED", LiveRefCount: 1, ChunkerVersion: "v2-fastcdc"}, + } + graphDigestA, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkChunkGraphRows(graphA)) + graphDigestB, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkChunkGraphRows(graphB)) + if graphDigestA != graphDigestB { + t.Fatalf("chunk graph digest changed with row order/IDs: %s != %s", graphDigestA, graphDigestB) + } + + physicalA := []benchmarkPhysicalRawRow{ + {BlockID: 10, ContainerID: 20, FormatVersion: 1, Codec: "aes-gcm", CompressionCodec: "none", BlockHash: "packed-a", ChunkHash: "chunk-a", SizeInBlock: sql.NullInt64{Int64: 1, Valid: true}}, + {BlockID: 11, ContainerID: 21, FormatVersion: 1, Codec: "aes-gcm", CompressionCodec: "none", BlockHash: "packed-b", ChunkHash: "chunk-b", SizeInBlock: sql.NullInt64{Int64: 2, Valid: true}}, + } + physicalB := []benchmarkPhysicalRawRow{ + {BlockID: 901, ContainerID: 801, FormatVersion: 1, Codec: "aes-gcm", CompressionCodec: "none", BlockHash: "different-packing-b", ChunkHash: "chunk-b", SizeInBlock: sql.NullInt64{Int64: 2, Valid: true}}, + {BlockID: 900, ContainerID: 800, FormatVersion: 1, Codec: "aes-gcm", CompressionCodec: "none", BlockHash: "different-packing-a", ChunkHash: "chunk-a", SizeInBlock: sql.NullInt64{Int64: 1, Valid: true}}, + } + physicalDigestA, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkPhysicalRows(physicalA)) + physicalDigestB, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkPhysicalRows(physicalB)) + if physicalDigestA != physicalDigestB { + t.Fatalf("canonical physical digest changed with IDs/packing: %s != %s", physicalDigestA, physicalDigestB) + } + physicalContentChanged := append([]benchmarkPhysicalRawRow(nil), physicalB...) + physicalContentChanged[0].ChunkHash = "changed-chunk-content" + changedDigest, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkPhysicalRows(physicalContentChanged)) + if physicalDigestA == changedDigest { + t.Fatal("canonical physical digest did not change with payload content") + } +} + +func TestBenchmarkDiagnosticRejectsDuplicateCanonicalLogicalPaths(t *testing.T) { + dataRoot := t.TempDir() + rows := []benchmarkActiveLogicalRawRow{ + {ID: 1, Path: filepath.Join(dataRoot, "duplicate.txt"), FileHash: "hash-a"}, + {ID: 2, Path: filepath.Join(dataRoot, "duplicate.txt"), FileHash: "hash-b"}, + } + if _, err := canonicalizeBenchmarkActiveLogicalRows(rows, dataRoot); err == nil { + t.Fatal("expected duplicate canonical logical path to fail") + } +} + +func TestBenchmarkDiagnosticSeparatesActiveNamespaceFromCatalogHistory(t *testing.T) { + dataRoot := t.TempDir() + active, err := canonicalizeBenchmarkActiveLogicalRows([]benchmarkActiveLogicalRawRow{ + {ID: 1, Path: filepath.Join(dataRoot, "active-a.txt"), FileHash: "hash-active", TotalSize: 10, Status: "COMPLETED", ChunkerVersion: "v2-fastcdc"}, + {ID: 1, Path: filepath.Join(dataRoot, "active-b.txt"), FileHash: "hash-active", TotalSize: 10, Status: "COMPLETED", ChunkerVersion: "v2-fastcdc"}, + }, dataRoot) + if err != nil { + t.Fatalf("canonicalize active namespace: %v", err) + } + if len(active) != 2 { + t.Fatalf("expected two active paths for one logical object, got %+v", active) + } + + catalog := canonicalizeBenchmarkLogicalCatalogRows([]benchmarkLogicalCatalogRawRow{ + {ID: 1, FileHash: "hash-active", TotalSize: 10, Status: "COMPLETED", RefCount: 2, ChunkerVersion: "v2-fastcdc", ActivePathCount: 2, SnapshotReferenceCount: 1}, + {ID: 2, FileHash: "hash-history", TotalSize: 20, Status: "COMPLETED", RefCount: 0, ChunkerVersion: "v2-fastcdc"}, + {ID: 3, FileHash: "hash-snapshot", TotalSize: 30, Status: "COMPLETED", RefCount: 0, ChunkerVersion: "v2-fastcdc", SnapshotReferenceCount: 2}, + }) + classes := make(map[string]string, len(catalog)) + for _, row := range catalog { + classes[row.FileHash] = row.ReachabilityClass + } + if classes["hash-active"] != "shared" || classes["hash-history"] != "unreachable_history" || classes["hash-snapshot"] != "snapshot_only" { + t.Fatalf("unexpected logical catalog reachability classes: %+v", classes) + } +} + +func TestBenchmarkDiagnosticLogicalQueriesSeparateNamespaceAndCatalog(t *testing.T) { + dbconn, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open logical diagnostic fixture: %v", err) + } + defer func() { _ = dbconn.Close() }() + dbconn.SetMaxOpenConns(1) + + for _, statement := range []string{ + `CREATE TABLE logical_file ( + id INTEGER PRIMARY KEY, original_name TEXT NOT NULL, file_hash TEXT NOT NULL, + total_size INTEGER NOT NULL, status TEXT NOT NULL, ref_count INTEGER NOT NULL, + chunker_version TEXT NOT NULL + )`, + `CREATE TABLE physical_file (path TEXT PRIMARY KEY, logical_file_id INTEGER NOT NULL)`, + `CREATE TABLE snapshot_file (snapshot_id TEXT NOT NULL, logical_file_id INTEGER NOT NULL)`, + `INSERT INTO logical_file VALUES + (101, 'sensitive-active-name', 'hash-active', 10, 'COMPLETED', 2, 'v2-fastcdc'), + (102, 'sensitive-history-name', 'hash-history', 20, 'COMPLETED', 0, 'v2-fastcdc'), + (103, 'sensitive-snapshot-name', 'hash-snapshot', 30, 'COMPLETED', 0, 'v2-fastcdc')`, + } { + if _, err := dbconn.Exec(statement); err != nil { + t.Fatalf("apply logical diagnostic fixture: %v", err) + } + } + dataRoot := t.TempDir() + if _, err := dbconn.Exec(`INSERT INTO physical_file VALUES (?, 101), (?, 101)`, + filepath.Join(dataRoot, "active-a.txt"), filepath.Join(dataRoot, "active-b.txt")); err != nil { + t.Fatalf("insert active namespace fixture: %v", err) + } + if _, err := dbconn.Exec(`INSERT INTO snapshot_file VALUES ('snapshot-a', 101), ('snapshot-a', 103)`); err != nil { + t.Fatalf("insert snapshot fixture: %v", err) + } + + activeRaw, activeBytes, err := readBenchmarkActiveLogicalNamespace(context.Background(), dbconn) + if err != nil { + t.Fatalf("read active logical namespace: %v", err) + } + active, err := canonicalizeBenchmarkActiveLogicalRows(activeRaw, dataRoot) + if err != nil { + t.Fatalf("canonicalize active logical namespace: %v", err) + } + if len(active) != 2 || activeBytes != 20 { + t.Fatalf("unexpected active namespace: rows=%+v bytes=%d", active, activeBytes) + } + + catalogRaw, catalogBytes, statuses, err := readBenchmarkLogicalCatalog(context.Background(), dbconn) + if err != nil { + t.Fatalf("read logical catalog: %v", err) + } + if len(catalogRaw) != 3 || catalogBytes != 60 || statuses.Completed != 3 { + t.Fatalf("unexpected logical catalog totals: rows=%+v bytes=%d statuses=%+v", catalogRaw, catalogBytes, statuses) + } + catalog := canonicalizeBenchmarkLogicalCatalogRows(catalogRaw) + classes := make(map[string]string, len(catalog)) + for _, row := range catalog { + classes[row.FileHash] = row.ReachabilityClass + } + if classes["hash-active"] != "shared" || classes["hash-history"] != "unreachable_history" || classes["hash-snapshot"] != "snapshot_only" { + t.Fatalf("unexpected query-backed catalog classes: %+v", classes) + } +} + +func TestBenchmarkDiagnosticGCAfterChurnPostgresEmitsCompleteV2State(t *testing.T) { + if strings.TrimSpace(os.Getenv("COLDKEEP_TEST_DB")) == "" { + t.Skip("set COLDKEEP_TEST_DB=1 with PostgreSQL DB_* variables to run the reduced gc-after-churn diagnostic test") + } + t.Setenv("COLDKEEP_DB_AUTO_BOOTSTRAP", "true") + databaseName, cleanup, err := createTemporaryBenchmarkDatabase("diagnostic-gc-after-churn") + if err != nil { + t.Fatalf("create reduced churn database: %v", err) + } + t.Cleanup(func() { + if err := cleanup(); err != nil { + t.Errorf("cleanup reduced churn database: %v", err) + } + }) + + connString, err := dbpkg.BuildPostgresConnStringFromEnv(databaseName) + if err != nil { + t.Fatalf("build reduced churn connection string: %v", err) + } + dbconn, err := sql.Open("postgres", connString) + if err != nil { + t.Fatalf("open reduced churn database: %v", err) + } + defer func() { _ = dbconn.Close() }() + if err := dbpkg.EnsureSchema(dbconn); err != nil { + t.Fatalf("bootstrap reduced churn schema: %v", err) + } + + insertLogical := func(name, fileHash string, refCount int64) int64 { + t.Helper() + var logicalID int64 + if err := dbconn.QueryRow(` + INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) + VALUES ($1, 1024, $2, 'COMPLETED', $3, 'v2-fastcdc') + RETURNING id + `, name, fileHash, refCount).Scan(&logicalID); err != nil { + t.Fatalf("insert reduced churn logical file: %v", err) + } + var chunkID int64 + if err := dbconn.QueryRow(` + INSERT INTO chunk (chunk_hash, size, status, live_ref_count, pin_count, chunker_version) + VALUES ($1, 1024, 'COMPLETED', 1, 0, 'v2-fastcdc') + RETURNING id + `, fileHash).Scan(&chunkID); err != nil { + t.Fatalf("insert reduced churn chunk: %v", err) + } + if _, err := dbconn.Exec(`INSERT INTO file_chunk (logical_file_id, chunk_id, chunk_order) VALUES ($1, $2, 0)`, logicalID, chunkID); err != nil { + t.Fatalf("insert reduced churn chunk graph: %v", err) + } + return logicalID + } + + dataRoot := t.TempDir() + activeID := insertLogical("sensitive-active-name", "active-content-hash", 1) + _ = insertLogical("sensitive-unreachable-history-name", "history-content-hash", 0) + activePath := filepath.Join(dataRoot, "churn", "active.bin") + if _, err := dbconn.Exec(`INSERT INTO physical_file (path, logical_file_id) VALUES ($1, $2)`, activePath, activeID); err != nil { + t.Fatalf("insert reduced churn active path: %v", err) + } + if _, err := dbconn.Exec(`INSERT INTO snapshot (id, created_at, type) VALUES ('reduced-churn', NOW(), 'full')`); err != nil { + t.Fatalf("insert reduced churn snapshot: %v", err) + } + var snapshotPathID int64 + if err := dbconn.QueryRow(`INSERT INTO snapshot_path (path) VALUES ('churn/active.bin') RETURNING id`).Scan(&snapshotPathID); err != nil { + t.Fatalf("insert reduced churn snapshot path: %v", err) + } + if _, err := dbconn.Exec(` + INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id, size) + VALUES ('reduced-churn', $1, $2, 1024) + `, snapshotPathID, activeID); err != nil { + t.Fatalf("insert reduced churn snapshot membership: %v", err) + } + + state, err := buildBenchmarkDiagnosticFinalState(context.Background(), dbconn, corebenchmark.BenchmarkContext{ + DataPath: dataRoot, + RepoPath: t.TempDir(), + }) + if err != nil { + t.Fatalf("build reduced gc-after-churn diagnostic state: %v", err) + } + if state.SchemaVersion != 2 || state.ActiveLogicalNamespace.Count != 1 || state.LogicalCatalog.Count != 2 || + state.LogicalStatuses.Completed != 2 || state.SnapshotCount != 1 || state.Snapshots.Count != 1 { + t.Fatalf("unexpected reduced gc-after-churn diagnostic state: %+v", state) + } + encoded, err := json.Marshal(state) + if err != nil { + t.Fatalf("encode reduced gc-after-churn diagnostic state: %v", err) + } + for _, forbidden := range []string{databaseName, dataRoot, "sensitive-active-name", "sensitive-unreachable-history-name"} { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("reduced gc-after-churn diagnostic leaked sensitive value %q: %s", forbidden, encoded) + } + } +} + +func TestBenchmarkDiagnosticLogicalDigestsChangeWithSemanticContent(t *testing.T) { + dataRoot := t.TempDir() + active := []benchmarkActiveLogicalRawRow{{ + ID: 1, Path: filepath.Join(dataRoot, "file.txt"), FileHash: "hash-a", TotalSize: 10, + Status: "COMPLETED", ChunkerVersion: "v2-fastcdc", + }} + firstActive, err := canonicalizeBenchmarkActiveLogicalRows(active, dataRoot) + if err != nil { + t.Fatalf("canonicalize active namespace: %v", err) + } + firstActiveDigest, _ := benchmarkCanonicalDigest(firstActive) + active[0].FileHash = "hash-b" + secondActive, err := canonicalizeBenchmarkActiveLogicalRows(active, dataRoot) + if err != nil { + t.Fatalf("canonicalize changed active namespace: %v", err) + } + secondActiveDigest, _ := benchmarkCanonicalDigest(secondActive) + if firstActiveDigest == secondActiveDigest { + t.Fatal("active namespace digest did not change with content") + } + + catalog := []benchmarkLogicalCatalogRawRow{{ + ID: 1, FileHash: "hash-a", TotalSize: 10, Status: "COMPLETED", RefCount: 1, + ChunkerVersion: "v2-fastcdc", ActivePathCount: 1, + }} + firstCatalogDigest, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkLogicalCatalogRows(catalog)) + catalog[0].ActivePathCount = 0 + catalog[0].RefCount = 0 + secondCatalogDigest, _ := benchmarkCanonicalDigest(canonicalizeBenchmarkLogicalCatalogRows(catalog)) + if firstCatalogDigest == secondCatalogDigest { + t.Fatal("logical catalog digest did not change with lifecycle state") + } +} + +func TestBenchmarkDiagnosticReportContainsNoSensitiveSourceValues(t *testing.T) { + dataRoot := t.TempDir() + sensitivePath := filepath.Join(dataRoot, "random-secret-name.txt") + canonical, err := canonicalizeBenchmarkActiveLogicalRows([]benchmarkActiveLogicalRawRow{{ + ID: 8675309, Path: sensitivePath, FileHash: "semantic-hash", TotalSize: 10, + Status: "COMPLETED", ChunkerVersion: "v2-fastcdc", + }}, dataRoot) + if err != nil { + t.Fatalf("canonicalize sensitive logical row: %v", err) + } + digest, err := benchmarkCanonicalDigest(canonical) + if err != nil { + t.Fatalf("digest sensitive logical row: %v", err) + } + encoded, err := json.Marshal(benchmarkDiagnosticFinalState{ + SchemaVersion: benchmarkDiagnosticFinalStateSchemaVersion, + ActiveLogicalNamespace: benchmarkDiagnosticDigest{Count: 1, TotalBytes: 10, SHA256: digest}, + LogicalCatalog: benchmarkDiagnosticDigest{Count: 1, TotalBytes: 10, SHA256: digest}, + }) + if err != nil { + t.Fatalf("marshal diagnostic final state: %v", err) + } + for _, forbidden := range []string{ + dataRoot, + "random-secret-name.txt", + "8675309", + "postgres://benchmark_user:benchmark_password@localhost/benchmark_database", + } { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("diagnostic report leaked sensitive value %q: %s", forbidden, encoded) + } + } +} + +func TestCanonicalBenchmarkActivePathRejectsEmptyAndTraversal(t *testing.T) { + dataRoot := t.TempDir() + if _, err := canonicalizeBenchmarkActiveLogicalRows([]benchmarkActiveLogicalRawRow{{Path: ""}}, dataRoot); err == nil { + t.Fatal("expected empty active path to fail") + } + outside := filepath.Join(filepath.Dir(dataRoot), "outside.txt") + if _, err := canonicalizeBenchmarkActiveLogicalRows([]benchmarkActiveLogicalRawRow{{Path: outside}}, dataRoot); err == nil { + t.Fatal("expected active path traversal to fail") + } +} + +func TestCanonicalBenchmarkSnapshotPathAcceptsRelativeAndRejectsTraversal(t *testing.T) { + dataRoot := t.TempDir() + got, err := canonicalBenchmarkSnapshotPath(dataRoot, "snapshot/nested/file.bin") + if err != nil || got != "snapshot/nested/file.bin" { + t.Fatalf("canonical relative snapshot path: got=%q err=%v", got, err) + } + if _, err := canonicalBenchmarkSnapshotPath(dataRoot, "../outside.txt"); err == nil { + t.Fatal("expected snapshot traversal to be rejected") + } + normalizedAbsolute := strings.TrimPrefix(filepath.ToSlash(filepath.Join(dataRoot, "snapshot", "file.bin")), "/") + got, err = canonicalBenchmarkSnapshotPath(dataRoot, normalizedAbsolute) + if err != nil || got != "snapshot/file.bin" { + t.Fatalf("canonical normalized absolute snapshot path: got=%q err=%v", got, err) + } +} + func (singleChunkV2CLITestChunker) Version() chunk.Version { return chunk.VersionV2FastCDC } @@ -107,10 +472,11 @@ func captureStdout(t *testing.T, fn func()) string { func runCLIWithCapturedIO(t *testing.T, args []string) (stdout string, stderr string, code int) { t.Helper() + runtime := newDefaultCommandTestRuntime(t) stderr = captureStderr(t, func() { stdout = captureStdout(t, func() { - code = runCLI(args) + code = runCLIWithRuntime(args, runtime) }) }) @@ -649,7 +1015,10 @@ func TestRunCLIRepairJSONFailureIncludesInvariantMetadata(t *testing.T) { } stderr := captureStderr(t, func() { - code := runCLI([]string{"repair", "ref-counts", "--output", "json"}) + code := runCLIWithRuntime( + []string{"repair", "ref-counts", "--output", "json"}, + newDefaultCommandTestRuntime(t), + ) if code != exitVerify { t.Fatalf("expected exit code %d, got %d", exitVerify, code) } @@ -782,7 +1151,10 @@ func TestRunCLIStoreJSONEmitsStartupRecoveryAndCrossVersionReuseSuccess(t *testi var stdout string stderr := captureStderr(t, func() { stdout = captureStdout(t, func() { - code := runCLI([]string{"store", "--output", "json", inPath}) + code := runCLIWithRuntime( + []string{"store", "--output", "json", inPath}, + newDefaultCommandTestRuntime(t), + ) if code != exitSuccess { t.Fatalf("expected exit code %d, got %d", exitSuccess, code) } @@ -2323,6 +2695,7 @@ func TestRunBenchmarkRunCommandJSONOutputSchema(t *testing.T) { t.Fatalf("unexpected default workers: %d", opts.StoreFolderWorkers) } return BenchmarkRunReport{ + SchemaVersion: 2, GeneratedAtUTC: "2026-04-29T00:00:00Z", Dataset: "small", Repeat: 2, @@ -2386,6 +2759,9 @@ func TestRunBenchmarkRunCommandJSONOutputSchema(t *testing.T) { if got, _ := data["dataset"].(string); got != "small" { t.Fatalf("dataset mismatch: got=%v data=%v", data["dataset"], data) } + if got, _ := data["schema_version"].(float64); int(got) != 2 { + t.Fatalf("schema_version mismatch: got=%v data=%v", data["schema_version"], data) + } if got, _ := data["repeat"].(float64); int(got) != 2 { t.Fatalf("repeat mismatch: got=%v data=%v", data["repeat"], data) } @@ -2445,6 +2821,78 @@ func TestRunBenchmarkRunCommandJSONOutputSchema(t *testing.T) { } } +func TestRunCLIBenchmarkJSONEmitsExactlyOneEnvelope(t *testing.T) { + originalPhase := runCoreBenchmarkPhase + t.Cleanup(func() { runCoreBenchmarkPhase = originalPhase }) + runCoreBenchmarkPhase = func( + preset corebenchmark.DatasetPreset, + repeat int, + opts execution.Options, + ) (BenchmarkRunReport, error) { + return BenchmarkRunReport{ + SchemaVersion: 2, + Dataset: string(preset), + Repeat: repeat, + Execution: BenchmarkExecution{ + StoreFolderWorkers: opts.StoreFolderWorkers, + PipelineDepth: opts.PipelineDepth, + Deterministic: opts.Deterministic, + }, + Rows: []BenchmarkRunCaseRow{{ + Case: "store-large-file", + DurationMs: 1000, + ThroughputMBps: 1, + }}, + }, nil + } + + output := captureStdout(t, func() { + if code := runCLI([]string{"benchmark", "run", "--output", "json"}); code != exitSuccess { + t.Fatalf("runCLI exit=%d", code) + } + }) + decoder := json.NewDecoder(strings.NewReader(output)) + var envelope map[string]any + if err := decoder.Decode(&envelope); err != nil { + t.Fatalf("decode benchmark envelope: %v; output=%q", err, output) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + t.Fatalf("expected exactly one JSON envelope, got trailing=%v err=%v output=%q", trailing, err, output) + } +} + +func TestRunCLIBenchmarkJSONFailureDoesNotEmitSuccess(t *testing.T) { + originalPhase := runCoreBenchmarkPhase + t.Cleanup(func() { runCoreBenchmarkPhase = originalPhase }) + runCoreBenchmarkPhase = func( + corebenchmark.DatasetPreset, + int, + execution.Options, + ) (BenchmarkRunReport, error) { + return BenchmarkRunReport{}, errors.New("benchmark failed") + } + + var stdout string + stderr := captureStderr(t, func() { + stdout = captureStdout(t, func() { + if code := runCLI([]string{"benchmark", "run", "--output", "json"}); code != exitGeneral { + t.Fatalf("runCLI exit=%d", code) + } + }) + }) + if strings.TrimSpace(stdout) != "" { + t.Fatalf("benchmark failure emitted success output: %q", stdout) + } + var envelope map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &envelope); err != nil { + t.Fatalf("decode benchmark error envelope: %v; stderr=%q", err, stderr) + } + if envelope["status"] != "error" { + t.Fatalf("expected error envelope, got=%v", envelope) + } +} + func TestRunBenchmarkRunCommandTableOutputIncludesRows(t *testing.T) { originalPhase := runCoreBenchmarkPhase t.Cleanup(func() { runCoreBenchmarkPhase = originalPhase }) @@ -3309,7 +3757,7 @@ func TestParseDoctorVerifyLevelUsesExplicitFlag(t *testing.T) { } func TestPrintCLISuccessJSONCommandPolicy(t *testing.T) { - selfEmittingJSONCommands := []string{"store", "store-folder", "restore", "remove", "repair", "gc", "list", "search", "stats", "inspect", "simulate", "doctor", "snapshot", "config", "version", "-v", "--version", "verify"} + selfEmittingJSONCommands := []string{"store", "store-folder", "restore", "remove", "repair", "gc", "list", "search", "stats", "inspect", "simulate", "benchmark", "doctor", "snapshot", "config", "version", "-v", "--version", "verify"} for _, command := range selfEmittingJSONCommands { output := captureStdout(t, func() { @@ -6082,6 +6530,16 @@ func TestRenderSnapshotTreeLinesDeterministicAcrossCalls(t *testing.T) { } } +func TestRenderSnapshotTreeLinesDeduplicatesDuplicateIDs(t *testing.T) { + lines := renderSnapshotTreeLines([]snapshot.Snapshot{ + {ID: "root", CreatedAt: time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC), Type: "full"}, + {ID: "root", CreatedAt: time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC), Type: "full"}, + }) + if got := strings.Join(lines, "\n"); got != "root" { + t.Fatalf("expected duplicate metadata records to render once, got %q", got) + } +} + func TestRunSnapshotCommandShowReturnsSnapshotAndFiles(t *testing.T) { originalLoad := loadDefaultStorageContextPhase originalGet := getSnapshotPhase @@ -8221,6 +8679,76 @@ func TestStatsCommandJSONShorthand(t *testing.T) { } } +func TestRunStatsCommandJSONPreservesExactLargeIntegers(t *testing.T) { + originalRunStats := runObservabilityStatsPhase + t.Cleanup(func() { runObservabilityStatsPhase = originalRunStats }) + + runObservabilityStatsPhase = func(observability.StatsOptions) (*observability.StatsResult, error) { + return &observability.StatsResult{ + Logical: observability.LogicalStats{ + TotalFiles: 9007199254740993, + TotalSizeBytes: math.MaxInt64, + }, + Chunks: observability.ChunkStats{ + ChunkerVersions: []observability.VersionStat{{ + Version: "v2-fastcdc", + Chunks: 9007199254740991, + Bytes: 9007199254740993, + }}, + }, + }, nil + } + + output := captureStdout(t, func() { + if err := runStatsCommand(parsedCommandLine{ + method: "stats", + flags: map[string][]string{"output": {"json"}}, + }, outputModeJSON); err != nil { + t.Fatalf("runStatsCommand JSON: %v", err) + } + }) + + var envelope struct { + Data struct { + Logical struct { + TotalFiles json.Number `json:"total_files"` + TotalSizeBytes json.Number `json:"total_size_bytes"` + } `json:"logical"` + Chunks struct { + ChunkerVersions []struct { + Chunks json.Number `json:"chunks"` + Bytes json.Number `json:"bytes"` + } `json:"chunker_versions"` + } `json:"chunks"` + } `json:"data"` + } + decoder := json.NewDecoder(strings.NewReader(output)) + decoder.UseNumber() + if err := decoder.Decode(&envelope); err != nil { + t.Fatalf("decode stats JSON: %v output=%q", err, output) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + t.Fatalf("stats JSON has trailing content: %v output=%q", err, output) + } + + if got := envelope.Data.Logical.TotalFiles.String(); got != "9007199254740993" { + t.Fatalf("data.logical.total_files=%q want 9007199254740993", got) + } + if got := envelope.Data.Logical.TotalSizeBytes.String(); got != "9223372036854775807" { + t.Fatalf("data.logical.total_size_bytes=%q want 9223372036854775807", got) + } + if len(envelope.Data.Chunks.ChunkerVersions) != 1 { + t.Fatalf("chunker_versions=%v want one entry", envelope.Data.Chunks.ChunkerVersions) + } + version := envelope.Data.Chunks.ChunkerVersions[0] + if got := version.Chunks.String(); got != "9007199254740991" { + t.Fatalf("chunker_versions[0].chunks=%q want 9007199254740991", got) + } + if got := version.Bytes.String(); got != "9007199254740993" { + t.Fatalf("chunker_versions[0].bytes=%q want 9007199254740993", got) + } +} + func TestStatsCommandContainers(t *testing.T) { originalRunStats := runObservabilityStatsPhase t.Cleanup(func() { runObservabilityStatsPhase = originalRunStats }) @@ -9915,17 +10443,14 @@ func TestJSONModeUsageErrorKeepsStdoutEmptyAndStderrJSONOnly(t *testing.T) { } stderrPayloads := assertEveryLineIsJSONObject(t, stderr) - if len(stderrPayloads) != 2 { - t.Fatalf("expected startup event + error payload on stderr, got %d lines output=%q", len(stderrPayloads), stderr) - } - if got, _ := stderrPayloads[0]["event"].(string); got != "startup_recovery" { - t.Fatalf("expected first stderr line to be startup_recovery event, got %v payload=%v", stderrPayloads[0]["event"], stderrPayloads[0]) + if len(stderrPayloads) != 1 { + t.Fatalf("expected only the parse-time error payload on stderr, got %d lines output=%q", len(stderrPayloads), stderr) } - if got, _ := stderrPayloads[1]["status"].(string); got != "error" { - t.Fatalf("expected second stderr line status=error, got %v payload=%v", stderrPayloads[1]["status"], stderrPayloads[1]) + if got, _ := stderrPayloads[0]["status"].(string); got != "error" { + t.Fatalf("expected stderr status=error, got %v payload=%v", stderrPayloads[0]["status"], stderrPayloads[0]) } - if got, _ := stderrPayloads[1]["error_class"].(string); got != "USAGE" { - t.Fatalf("expected second stderr line error_class=USAGE, got %v payload=%v", stderrPayloads[1]["error_class"], stderrPayloads[1]) + if got, _ := stderrPayloads[0]["error_class"].(string); got != "USAGE" { + t.Fatalf("expected stderr error_class=USAGE, got %v payload=%v", stderrPayloads[0]["error_class"], stderrPayloads[0]) } } @@ -10157,7 +10682,7 @@ func TestCompareWithBaselineThroughputRegression(t *testing.T) { } } -func TestCompareWithBaselineNewCaseIgnored(t *testing.T) { +func TestCompareWithBaselineRejectsUnexpectedCase(t *testing.T) { baseline := BenchmarkRunReport{ Rows: []BenchmarkRunCaseRow{ {Case: "store-large", DurationMs: 1000, ThroughputMBps: 200}, @@ -10171,8 +10696,60 @@ func TestCompareWithBaselineNewCaseIgnored(t *testing.T) { } baselineFile := writeBaselineJSON(t, baseline) - if err := compareWithBaseline(current, baselineFile, 20.0); err != nil { - t.Fatalf("expected no regression for new cases, got: %v", err) + if err := compareWithBaseline(current, baselineFile, 20.0); err == nil || !strings.Contains(err.Error(), "case set mismatch") { + t.Fatalf("expected case-set validation error, got: %v", err) + } +} + +func TestCompareWithBaselineRejectsDuplicateCase(t *testing.T) { + baseline := BenchmarkRunReport{ + Rows: []BenchmarkRunCaseRow{ + {Case: "store-large", DurationMs: 1000, ThroughputMBps: 200}, + {Case: "store-large", DurationMs: 1001, ThroughputMBps: 199}, + }, + } + current := BenchmarkRunReport{ + Rows: []BenchmarkRunCaseRow{ + {Case: "store-large", DurationMs: 1000, ThroughputMBps: 200}, + {Case: "restore-large", DurationMs: 1000, ThroughputMBps: 200}, + }, + } + err := compareWithBaseline(current, writeBaselineJSON(t, baseline), 20) + if err == nil || !strings.Contains(err.Error(), "duplicate case") { + t.Fatalf("expected duplicate-case validation error, got: %v", err) + } +} + +func TestRunBenchmarkRunCommandRejectsRepeatForCIStableFixture(t *testing.T) { + err := runBenchmarkCommand(parsedCommandLine{ + method: "benchmark", + positionals: []string{"run"}, + flags: map[string][]string{ + "dataset": {"ci-stable-v1"}, + "repeat": {"2"}, + }, + }, outputModeJSON) + if err == nil || !strings.Contains(err.Error(), "requires --repeat 1") { + t.Fatalf("expected ci-stable repeat error, got: %v", err) + } +} + +func TestRunBenchmarkRunCommandRejectsRepeatForPairedFixtures(t *testing.T) { + for _, dataset := range []string{ + "ci-paired-w1-v1", "ci-paired-w4-v1", + "ci-paired-w1-v2", "ci-paired-w4-v2", + } { + err := runBenchmarkCommand(parsedCommandLine{ + method: "benchmark", + positionals: []string{"run"}, + flags: map[string][]string{ + "dataset": {dataset}, + "repeat": {"2"}, + }, + }, outputModeJSON) + if err == nil || !strings.Contains(err.Error(), "requires --repeat 1") { + t.Fatalf("expected paired repeat error for %q, got: %v", dataset, err) + } } } diff --git a/cmd/coldkeep/partial_routing_boundaries_test.go b/cmd/coldkeep/partial_routing_boundaries_test.go index cd842d03..9b4f35f5 100644 --- a/cmd/coldkeep/partial_routing_boundaries_test.go +++ b/cmd/coldkeep/partial_routing_boundaries_test.go @@ -111,7 +111,7 @@ func TestRunSnapshotCommandShowPreservesMixedOwnershipSeams(t *testing.T) { } } -func TestDiffSnapshotsPhaseNarrowsPrefixesBeforeEngineSeam(t *testing.T) { +func TestDiffSnapshotsPhasePreservesRepeatedPathsAndPrefixes(t *testing.T) { dbconn := openSnapshotRoutingDB(t) now := time.Now().UTC().Truncate(time.Second) @@ -134,12 +134,25 @@ func TestDiffSnapshotsPhaseNarrowsPrefixesBeforeEngineSeam(t *testing.T) { paths := make([]string, 0, len(result.Entries)) for _, entry := range result.Entries { paths = append(paths, entry.Path) - if strings.HasPrefix(entry.Path, "images/") { - t.Fatalf("expected diff seam to narrow to first prefix only, got image entry %q", entry.Path) - } } - if len(paths) != 2 { - t.Fatalf("expected only docs/ entries after narrowing, got %v", paths) + if got, want := strings.Join(paths, ","), "docs/added.txt,docs/removed.txt,images/added.png,images/removed.png"; got != want { + t.Fatalf("expected repeated prefixes to preserve all matching entries, got %v", paths) + } + + query = &snapshot.SnapshotQuery{ExactPaths: map[string]struct{}{ + "docs/added.txt": {}, + "images/removed.png": {}, + }} + result, err = diffSnapshotsPhase(context.Background(), dbconn, "diff-narrow-base", "diff-narrow-target", query) + if err != nil { + t.Fatalf("diffSnapshotsPhase repeated paths: %v", err) + } + paths = paths[:0] + for _, entry := range result.Entries { + paths = append(paths, entry.Path) + } + if got, want := strings.Join(paths, ","), "docs/added.txt,images/removed.png"; got != want { + t.Fatalf("expected repeated paths to preserve both matching entries, got %v", paths) } } diff --git a/cmd/coldkeep/repository_coordination_contract.go b/cmd/coldkeep/repository_coordination_contract.go new file mode 100644 index 00000000..a63a3937 --- /dev/null +++ b/cmd/coldkeep/repository_coordination_contract.go @@ -0,0 +1,92 @@ +package main + +import ( + "strings" + + "github.com/franchoy/coldkeep/internal/coordination" +) + +// repositoryCoordinationPolicy is the Phase 11 command classification consumed +// by the single top-level repository lease wrapper. +type repositoryCoordinationPolicy struct { + Required bool + Operation coordination.Operation + Mode coordination.Mode +} + +var repositoryOperations = map[string]coordination.Operation{ + "store": coordination.OperationStore, + "store-folder": coordination.OperationStoreFolder, + "restore": coordination.OperationRestore, + "remove": coordination.OperationRemove, + "repair": coordination.OperationRepair, + "gc": coordination.OperationGarbageCollect, + "stats": coordination.OperationStats, + "inspect": coordination.OperationInspect, + "list": coordination.OperationList, + "search": coordination.OperationSearch, + "verify": coordination.OperationVerify, + "doctor": coordination.OperationDoctor, +} + +var snapshotOperations = map[string]coordination.Operation{ + "create": coordination.OperationSnapshotCreate, + "delete": coordination.OperationSnapshotDelete, + "restore": coordination.OperationSnapshotRestore, + "list": coordination.OperationSnapshotList, + "show": coordination.OperationSnapshotShow, + "stats": coordination.OperationSnapshotStats, + "diff": coordination.OperationSnapshotDiff, +} + +func repositoryCoordinationPolicyFor(parsed parsedCommandLine) repositoryCoordinationPolicy { + if parsed.hasFlag("help", "h") { + return repositoryCoordinationPolicy{} + } + + if operation, ok := repositoryOperations[parsed.method]; ok { + return exclusiveRepositoryPolicy(operation) + } + switch parsed.method { + case "config": + return configRepositoryPolicy(parsed.positionals) + case "snapshot": + return snapshotRepositoryPolicy(parsed.positionals) + } + // init, simulate, benchmark, version, help, and invalid commands do not + // access the shared repository through this policy. + return repositoryCoordinationPolicy{} +} + +func exclusiveRepositoryPolicy(operation coordination.Operation) repositoryCoordinationPolicy { + return repositoryCoordinationPolicy{ + Required: true, + Operation: operation, + Mode: coordination.ModeExclusive, + } +} + +func configRepositoryPolicy(positionals []string) repositoryCoordinationPolicy { + if len(positionals) == 0 { + return repositoryCoordinationPolicy{} + } + switch strings.ToLower(strings.TrimSpace(positionals[0])) { + case "get": + return exclusiveRepositoryPolicy(coordination.OperationConfigGet) + case "set": + return exclusiveRepositoryPolicy(coordination.OperationConfigSet) + default: + return repositoryCoordinationPolicy{} + } +} + +func snapshotRepositoryPolicy(positionals []string) repositoryCoordinationPolicy { + if len(positionals) == 0 { + return repositoryCoordinationPolicy{} + } + operation, ok := snapshotOperations[strings.ToLower(strings.TrimSpace(positionals[0]))] + if !ok { + return repositoryCoordinationPolicy{} + } + return exclusiveRepositoryPolicy(operation) +} diff --git a/cmd/coldkeep/repository_coordination_contract_test.go b/cmd/coldkeep/repository_coordination_contract_test.go new file mode 100644 index 00000000..e246194c --- /dev/null +++ b/cmd/coldkeep/repository_coordination_contract_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "testing" + + "github.com/franchoy/coldkeep/internal/coordination" +) + +func TestRepositoryCoordinationPolicyRequiresExclusiveRepositoryOperations(t *testing.T) { + tests := []struct { + name string + method string + positionals []string + operation coordination.Operation + }{ + {name: "store", method: "store", operation: coordination.OperationStore}, + {name: "store folder", method: "store-folder", operation: coordination.OperationStoreFolder}, + {name: "restore", method: "restore", operation: coordination.OperationRestore}, + {name: "remove", method: "remove", operation: coordination.OperationRemove}, + {name: "repair", method: "repair", operation: coordination.OperationRepair}, + {name: "gc", method: "gc", operation: coordination.OperationGarbageCollect}, + {name: "stats", method: "stats", operation: coordination.OperationStats}, + {name: "inspect", method: "inspect", operation: coordination.OperationInspect}, + {name: "list", method: "list", operation: coordination.OperationList}, + {name: "search", method: "search", operation: coordination.OperationSearch}, + {name: "verify", method: "verify", operation: coordination.OperationVerify}, + {name: "doctor", method: "doctor", operation: coordination.OperationDoctor}, + {name: "config get", method: "config", positionals: []string{"get", "compression"}, operation: coordination.OperationConfigGet}, + {name: "config set", method: "config", positionals: []string{"set", "compression", "zstd"}, operation: coordination.OperationConfigSet}, + {name: "snapshot create", method: "snapshot", positionals: []string{"create"}, operation: coordination.OperationSnapshotCreate}, + {name: "snapshot delete", method: "snapshot", positionals: []string{"delete"}, operation: coordination.OperationSnapshotDelete}, + {name: "snapshot restore", method: "snapshot", positionals: []string{"restore"}, operation: coordination.OperationSnapshotRestore}, + {name: "snapshot list", method: "snapshot", positionals: []string{"list"}, operation: coordination.OperationSnapshotList}, + {name: "snapshot show", method: "snapshot", positionals: []string{"show"}, operation: coordination.OperationSnapshotShow}, + {name: "snapshot stats", method: "snapshot", positionals: []string{"stats"}, operation: coordination.OperationSnapshotStats}, + {name: "snapshot diff", method: "snapshot", positionals: []string{"diff"}, operation: coordination.OperationSnapshotDiff}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + policy := repositoryCoordinationPolicyFor(parsedCommandLine{ + method: tt.method, + positionals: tt.positionals, + flags: map[string][]string{}, + }) + if !policy.Required { + t.Fatal("repository coordination was not required") + } + if policy.Mode != coordination.ModeExclusive { + t.Fatalf("mode=%q want=%q", policy.Mode, coordination.ModeExclusive) + } + if policy.Operation != tt.operation { + t.Fatalf("operation=%q want=%q", policy.Operation, tt.operation) + } + }) + } +} + +func TestRepositoryCoordinationPolicyKeepsRestoreVerifyAndBothGCModesExclusive(t *testing.T) { + tests := []struct { + name string + parsed parsedCommandLine + operation coordination.Operation + }{ + {name: "restore", parsed: parsedCommandLine{method: "restore"}, operation: coordination.OperationRestore}, + {name: "verify fast", parsed: parsedCommandLine{method: "verify", flags: map[string][]string{"fast": {""}}}, operation: coordination.OperationVerify}, + {name: "verify standard", parsed: parsedCommandLine{method: "verify", flags: map[string][]string{"standard": {""}}}, operation: coordination.OperationVerify}, + {name: "verify full", parsed: parsedCommandLine{method: "verify", flags: map[string][]string{"full": {""}}}, operation: coordination.OperationVerify}, + {name: "verify deep", parsed: parsedCommandLine{method: "verify", flags: map[string][]string{"deep": {""}}}, operation: coordination.OperationVerify}, + {name: "gc dry run", parsed: parsedCommandLine{method: "gc", flags: map[string][]string{"dry-run": {""}}}, operation: coordination.OperationGarbageCollect}, + {name: "gc live", parsed: parsedCommandLine{method: "gc"}, operation: coordination.OperationGarbageCollect}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.parsed.flags == nil { + test.parsed.flags = map[string][]string{} + } + policy := repositoryCoordinationPolicyFor(test.parsed) + if !policy.Required || policy.Mode != coordination.ModeExclusive || policy.Operation != test.operation { + t.Fatalf("policy=%+v want required exclusive operation=%q", policy, test.operation) + } + }) + } +} + +func TestRepositoryCoordinationPolicyBypassesNonRepositoryCommands(t *testing.T) { + tests := []parsedCommandLine{ + {method: "init"}, + {method: "simulate", positionals: []string{"store"}}, + {method: "benchmark", positionals: []string{"run"}}, + {method: "version"}, + {method: "help"}, + {method: "-h"}, + {method: "--help"}, + {method: "-v"}, + {method: "--version"}, + {method: ""}, + {method: "unknown"}, + {method: "snapshot"}, + {method: "snapshot", positionals: []string{"unknown"}}, + {method: "config"}, + {method: "config", positionals: []string{"unknown"}}, + } + + for _, parsed := range tests { + parsed.flags = map[string][]string{} + policy := repositoryCoordinationPolicyFor(parsed) + if policy.Required || policy.Operation != "" || policy.Mode != "" { + t.Fatalf("method=%q positionals=%v unexpectedly coordinated: %+v", parsed.method, parsed.positionals, policy) + } + } +} + +func TestRepositoryCoordinationPolicyUsesParsedHelpAndDoubleDashSemantics(t *testing.T) { + tests := []struct { + name string + args []string + required bool + want coordination.Operation + }{ + {name: "help before command", args: []string{"--help", "store"}}, + {name: "command help", args: []string{"store", "--help"}}, + {name: "snapshot subcommand after double dash", args: []string{"snapshot", "--", "create"}, required: true, want: coordination.OperationSnapshotCreate}, + {name: "config subcommand after double dash", args: []string{"config", "--", "get", "compression"}, required: true, want: coordination.OperationConfigGet}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parsed, err := parseCommandLine(tt.args, flagsWithValues) + if err != nil { + t.Fatalf("parseCommandLine(%v): %v", tt.args, err) + } + policy := repositoryCoordinationPolicyFor(parsed) + if policy.Required != tt.required || policy.Operation != tt.want { + t.Fatalf("policy=%+v required=%v operation=%q", policy, tt.required, tt.want) + } + if tt.required && policy.Mode != coordination.ModeExclusive { + t.Fatalf("mode=%q want=%q", policy.Mode, coordination.ModeExclusive) + } + }) + } +} + +func TestRepositoryCoordinationPolicyBypassesCommandHelp(t *testing.T) { + for _, flag := range []string{"help", "h"} { + policy := repositoryCoordinationPolicyFor(parsedCommandLine{ + method: "store", + flags: map[string][]string{flag: {""}}, + }) + if policy.Required { + t.Fatalf("--%s unexpectedly required coordination", flag) + } + } +} diff --git a/cmd/coldkeep/repository_coordination_errors.go b/cmd/coldkeep/repository_coordination_errors.go new file mode 100644 index 00000000..a385e8ba --- /dev/null +++ b/cmd/coldkeep/repository_coordination_errors.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "strings" + + "github.com/franchoy/coldkeep/internal/coordination" +) + +const ( + publicCodeRepositoryBusy = "REPOSITORY_BUSY" + publicCodeRepositoryLockUnsupported = "REPOSITORY_LOCK_UNSUPPORTED" + publicCodePermissionDenied = "PERMISSION_DENIED" + publicCodeCanceled = "CANCELED" + publicCodeDeadlineExceeded = "DEADLINE_EXCEEDED" +) + +// repositoryCoordinationFailure marks an error produced while acquiring or +// releasing the outer repository lease. It preserves the original cause for +// errors.Is/errors.As while allowing generic I/O failures to use a safe CLI +// message rather than exposing repository paths or native-lock details. +type repositoryCoordinationFailure struct { + err error +} + +func (failure *repositoryCoordinationFailure) Error() string { + return failure.err.Error() +} + +func (failure *repositoryCoordinationFailure) Unwrap() error { + return failure.err +} + +func markRepositoryCoordinationFailure(err error) error { + if err == nil { + return nil + } + var marked *repositoryCoordinationFailure + if errors.As(err, &marked) { + return err + } + return &repositoryCoordinationFailure{err: err} +} + +// cliRepositoryCoordinator keeps the coordination stage visible to the CLI +// classifier without changing the internal coordination contract. +type cliRepositoryCoordinator struct { + delegate coordination.Coordinator +} + +func (coordinator cliRepositoryCoordinator) Acquire( + ctx context.Context, + identity coordination.Identity, + request coordination.Request, +) (coordination.Lease, error) { + if coordinator.delegate == nil { + return nil, markRepositoryCoordinationFailure(fmt.Errorf("repository coordinator is unavailable")) + } + lease, err := coordinator.delegate.Acquire(ctx, identity, request) + if err != nil { + return nil, markRepositoryCoordinationFailure(err) + } + if lease == nil { + return nil, nil + } + return &cliRepositoryLease{delegate: lease}, nil +} + +type cliRepositoryLease struct { + delegate coordination.Lease +} + +func (lease *cliRepositoryLease) Release() error { + if lease == nil || lease.delegate == nil { + return nil + } + return markRepositoryCoordinationFailure(lease.delegate.Release()) +} + +// stableCLIError maps established coordination and runtime causes into the +// existing cliError representation. Existing explicit cliError ownership wins, +// which preserves the operation-first precedence of joined operation/release +// failures produced by coordination.WithLease. +func stableCLIError(err error) error { + if err == nil { + return nil + } + if joinedError, ok := stableJoinedCLIError(err); ok { + return joinedError + } + + var existing *cliError + if errors.As(err, &existing) { + return err + } + + if classified, ok := stableSentinelCLIError(err); ok { + return classified + } + + var coordinationFailure *repositoryCoordinationFailure + if errors.As(err, &coordinationFailure) { + return observabilityWrappedError(exitGeneral, "INTERNAL", "repository coordination failed", err) + } + + return err +} + +func stableJoinedCLIError(err error) (error, bool) { + joined, ok := err.(interface{ Unwrap() []error }) + if !ok || !hasStableCLIClassification(err) { + return nil, false + } + causes := joined.Unwrap() + if len(causes) == 0 { + return nil, false + } + primary := stableCLIError(causes[0]) + var classified *cliError + if errors.As(primary, &classified) { + return &cliError{code: classified.code, msg: classified.msg, err: err, publicCode: classified.publicCode}, true + } + return observabilityWrappedError(exitGeneral, "INTERNAL", strings.TrimSpace(causes[0].Error()), err), true +} + +func stableSentinelCLIError(err error) (error, bool) { + switch { + case errors.Is(err, coordination.ErrRepositoryIdentityInvalid): + return observabilityWrappedError(exitUsage, "INVALID_ARGUMENT", "repository identity is invalid", err), true + case errors.Is(err, coordination.ErrRepositoryBusy): + return observabilityWrappedError(exitGeneral, publicCodeRepositoryBusy, "repository is busy", err), true + case errors.Is(err, coordination.ErrRepositoryLockUnsupported): + return observabilityWrappedError(exitGeneral, publicCodeRepositoryLockUnsupported, "repository locking is unsupported", err), true + case errors.Is(err, coordination.ErrNestedRepositoryAcquisition): + return observabilityWrappedError(exitGeneral, "INTERNAL", "repository coordination failed", err), true + case errors.Is(err, context.Canceled): + return observabilityWrappedError(exitGeneral, publicCodeCanceled, "operation canceled", err), true + case errors.Is(err, context.DeadlineExceeded): + return observabilityWrappedError(exitGeneral, publicCodeDeadlineExceeded, "operation deadline exceeded", err), true + case errors.Is(err, fs.ErrPermission): + return observabilityWrappedError(exitGeneral, publicCodePermissionDenied, "permission denied", err), true + } + return nil, false +} + +func hasStableCLIClassification(err error) bool { + if err == nil { + return false + } + if stableSentinelCLIClassification(err) { + return true + } + var coordinationFailure *repositoryCoordinationFailure + return errors.As(err, &coordinationFailure) +} + +func stableSentinelCLIClassification(err error) bool { + _, ok := stableSentinelCLIError(err) + return ok +} diff --git a/cmd/coldkeep/repository_coordination_errors_test.go b/cmd/coldkeep/repository_coordination_errors_test.go new file mode 100644 index 00000000..1a294852 --- /dev/null +++ b/cmd/coldkeep/repository_coordination_errors_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/coordination" +) + +func TestStableCoordinationErrorMapping(t *testing.T) { + unexpectedIO := errors.New("unexpected native I/O") + tests := []struct { + name string + err error + wantCause error + wantCode string + wantExit int + wantMessage string + }{ + { + name: "busy wrapped", + err: fmt.Errorf("acquire: %w", coordination.ErrRepositoryBusy), + wantCause: coordination.ErrRepositoryBusy, + wantCode: publicCodeRepositoryBusy, + wantExit: exitGeneral, + wantMessage: "repository is busy", + }, + { + name: "unsupported wrapped", + err: fmt.Errorf("acquire: %w", coordination.ErrRepositoryLockUnsupported), + wantCause: coordination.ErrRepositoryLockUnsupported, + wantCode: publicCodeRepositoryLockUnsupported, + wantExit: exitGeneral, + wantMessage: "repository locking is unsupported", + }, + { + name: "identity invalid wrapped", + err: fmt.Errorf("resolve: %w", coordination.ErrRepositoryIdentityInvalid), + wantCause: coordination.ErrRepositoryIdentityInvalid, + wantCode: "INVALID_ARGUMENT", + wantExit: exitUsage, + wantMessage: "repository identity is invalid", + }, + { + name: "nested acquisition", + err: coordination.ErrNestedRepositoryAcquisition, + wantCause: coordination.ErrNestedRepositoryAcquisition, + wantCode: "INTERNAL", + wantExit: exitGeneral, + wantMessage: "repository coordination failed", + }, + { + name: "wrapped permission", + err: fmt.Errorf("open repository lock: %w", os.ErrPermission), + wantCause: os.ErrPermission, + wantCode: publicCodePermissionDenied, + wantExit: exitGeneral, + wantMessage: "permission denied", + }, + { + name: "context canceled", + err: fmt.Errorf("acquire: %w", context.Canceled), + wantCause: context.Canceled, + wantCode: publicCodeCanceled, + wantExit: exitGeneral, + wantMessage: "operation canceled", + }, + { + name: "context deadline", + err: fmt.Errorf("acquire: %w", context.DeadlineExceeded), + wantCause: context.DeadlineExceeded, + wantCode: publicCodeDeadlineExceeded, + wantExit: exitGeneral, + wantMessage: "operation deadline exceeded", + }, + { + name: "unexpected coordination I/O", + err: markRepositoryCoordinationFailure(unexpectedIO), + wantCause: unexpectedIO, + wantCode: "INTERNAL", + wantExit: exitGeneral, + wantMessage: "repository coordination failed", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + stableErr := stableCLIError(test.err) + if !errors.Is(stableErr, test.wantCause) { + t.Fatalf("stable error=%v does not preserve cause %v", stableErr, test.wantCause) + } + if got := classifyExitCode(stableErr); got != test.wantExit { + t.Fatalf("exit=%d want=%d", got, test.wantExit) + } + if got := publicErrorCode(stableErr, test.wantExit); got != test.wantCode { + t.Fatalf("public code=%q want=%q", got, test.wantCode) + } + if got := stableErr.Error(); got != test.wantMessage { + t.Fatalf("message=%q want=%q", got, test.wantMessage) + } + }) + } +} + +func TestStableCoordinationJoinedErrorPreservesOperationPrecedenceAndCauses(t *testing.T) { + operationCause := errors.New("invalid operation input") + operationErr := observabilityWrappedError(exitUsage, "INVALID_ARGUMENT", "invalid request", operationCause) + releaseCause := errors.New("native release failure") + releaseErr := markRepositoryCoordinationFailure(releaseCause) + joinedErr := errors.Join(operationErr, releaseErr) + + stableErr := stableCLIError(joinedErr) + if !errors.Is(stableErr, operationCause) || !errors.Is(stableErr, releaseCause) { + t.Fatalf("stable error lost joined causes: %v", stableErr) + } + if got := classifyExitCode(stableErr); got != exitUsage { + t.Fatalf("exit=%d want=%d", got, exitUsage) + } + if got := publicErrorCode(stableErr, exitUsage); got != "INVALID_ARGUMENT" { + t.Fatalf("public code=%q want=INVALID_ARGUMENT", got) + } +} + +func TestStableCoordinationJoinedSentinelUsesFirstCause(t *testing.T) { + secondaryCause := errors.New("secondary acquisition detail") + joinedErr := errors.Join( + fmt.Errorf("acquire: %w", coordination.ErrRepositoryBusy), + secondaryCause, + ) + + stableErr := stableCLIError(joinedErr) + if !errors.Is(stableErr, coordination.ErrRepositoryBusy) || !errors.Is(stableErr, secondaryCause) { + t.Fatalf("stable error lost joined causes: %v", stableErr) + } + if got := classifyExitCode(stableErr); got != exitGeneral { + t.Fatalf("exit=%d want=%d", got, exitGeneral) + } + if got := publicErrorCode(stableErr, exitGeneral); got != publicCodeRepositoryBusy { + t.Fatalf("public code=%q want=%q", got, publicCodeRepositoryBusy) + } +} + +func TestCoordinationJSONErrorRenderingUsesStableCodesAndSafeMessages(t *testing.T) { + tests := []struct { + name string + err error + wantCode string + }{ + { + name: "busy", + err: fmt.Errorf("lock /secret/repository/.coldkeep-control/repository.lock: %w", coordination.ErrRepositoryBusy), + wantCode: publicCodeRepositoryBusy, + }, + { + name: "unsupported", + err: fmt.Errorf("lock /secret/repository/.coldkeep-control/repository.lock: %w", coordination.ErrRepositoryLockUnsupported), + wantCode: publicCodeRepositoryLockUnsupported, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var code int + stderr := captureStderr(t, func() { + code = printCLIError(test.err, outputModeJSON) + }) + if code != exitGeneral { + t.Fatalf("exit=%d want=%d", code, exitGeneral) + } + if strings.Contains(stderr, "/secret/repository") { + t.Fatalf("JSON error leaked repository path: %s", stderr) + } + var payload map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &payload); err != nil { + t.Fatalf("decode JSON error: %v output=%q", err, stderr) + } + errorNode, ok := payload["error"].(map[string]any) + if !ok { + t.Fatalf("error node=%T want object", payload["error"]) + } + if got, _ := errorNode["code"].(string); got != test.wantCode { + t.Fatalf("code=%q want=%q payload=%v", got, test.wantCode, payload) + } + }) + } +} + +func TestCoordinationTextErrorRenderingUsesSafeMessage(t *testing.T) { + err := fmt.Errorf("lock /secret/repository/.coldkeep-control/repository.lock: %w", coordination.ErrRepositoryBusy) + var code int + stderr := captureStderr(t, func() { + code = printCLIError(err, outputModeText) + }) + if code != exitGeneral { + t.Fatalf("exit=%d want=%d", code, exitGeneral) + } + if strings.Contains(stderr, "/secret/repository") { + t.Fatalf("text error leaked repository path: %s", stderr) + } + if !strings.Contains(stderr, "repository is busy") { + t.Fatalf("text error=%q does not contain stable busy message", stderr) + } +} + +func TestRunCLICoordinationAcquisitionErrorsUseStableSurfaceAndShortCircuit(t *testing.T) { + unexpectedIO := errors.New("native coordination I/O failure") + tests := []struct { + name string + err error + wantCode string + wantExit int + }{ + {name: "busy", err: fmt.Errorf("acquire: %w", coordination.ErrRepositoryBusy), wantCode: publicCodeRepositoryBusy, wantExit: exitGeneral}, + {name: "unsupported", err: fmt.Errorf("acquire: %w", coordination.ErrRepositoryLockUnsupported), wantCode: publicCodeRepositoryLockUnsupported, wantExit: exitGeneral}, + {name: "identity invalid", err: fmt.Errorf("resolve: %w", coordination.ErrRepositoryIdentityInvalid), wantCode: "INVALID_ARGUMENT", wantExit: exitUsage}, + {name: "nested", err: coordination.ErrNestedRepositoryAcquisition, wantCode: "INTERNAL", wantExit: exitGeneral}, + {name: "permission", err: fmt.Errorf("open lock: %w", os.ErrPermission), wantCode: publicCodePermissionDenied, wantExit: exitGeneral}, + {name: "canceled", err: context.Canceled, wantCode: publicCodeCanceled, wantExit: exitGeneral}, + {name: "deadline", err: context.DeadlineExceeded, wantCode: publicCodeDeadlineExceeded, wantExit: exitGeneral}, + {name: "unexpected I/O", err: unexpectedIO, wantCode: "INTERNAL", wantExit: exitGeneral}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return nil, test.err + }, + } + } + + stdout, stderr, code := captureRuntimeCLI(t, []string{"stats", "--output", "json"}, runtime) + if code != test.wantExit { + t.Fatalf("exit=%d want=%d stderr=%q", code, test.wantExit, stderr) + } + if strings.TrimSpace(stdout) != "" { + t.Fatalf("stdout=%q want empty", stdout) + } + var payload map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &payload); err != nil { + t.Fatalf("decode JSON error: %v output=%q", err, stderr) + } + errorNode, ok := payload["error"].(map[string]any) + if !ok { + t.Fatalf("error node=%T want object", payload["error"]) + } + if got, _ := errorNode["code"].(string); got != test.wantCode { + t.Fatalf("code=%q want=%q payload=%v", got, test.wantCode, payload) + } + trace.require(t, []string{"lease acquire"}) + }) + } +} diff --git a/cmd/coldkeep/repository_coordination_runtime_contract_test.go b/cmd/coldkeep/repository_coordination_runtime_contract_test.go new file mode 100644 index 00000000..129044fa --- /dev/null +++ b/cmd/coldkeep/repository_coordination_runtime_contract_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/coordination" +) + +func TestRunCLIRestoreVerifyAndGCRuntimeWithinLease(t *testing.T) { + tests := []struct { + name string + args []string + operation coordination.Operation + event string + }{ + {name: "restore", args: []string{"restore", "42"}, operation: coordination.OperationRestore, event: "restore runtime"}, + {name: "verify", args: []string{"verify", "system"}, operation: coordination.OperationVerify, event: "verify runtime"}, + {name: "gc dry run", args: []string{"gc", "--dry-run"}, operation: coordination.OperationGarbageCollect, event: "gc dry-run runtime"}, + {name: "gc live", args: []string{"gc"}, operation: coordination.OperationGarbageCollect, event: "gc live runtime"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + var acquiredOperation coordination.Operation + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(_ context.Context, _ coordination.Identity, request coordination.Request) (coordination.Lease, error) { + acquiredOperation = request.Operation + trace.add("lease acquire") + return &fakeCLILease{releaseFn: func() error { + trace.add("lease release") + return nil + }}, nil + }, + } + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add(test.event) + defer trace.add(test.event + " cleanup") + return nil + } + + _, stderr, code := captureRuntimeCLI(t, test.args, runtime) + if code != exitSuccess { + t.Fatalf("exit=%d want=%d stderr=%q", code, exitSuccess, stderr) + } + if acquiredOperation != test.operation { + t.Fatalf("acquired operation=%q want=%q", acquiredOperation, test.operation) + } + trace.require(t, []string{ + "lease acquire", + "recovery", + test.event, + test.event + " cleanup", + "lease release", + "render success", + }) + }) + } +} + +func TestRunCLIRestoreVerifyAndGCFailuresCleanUpBeforeLeaseRelease(t *testing.T) { + tests := []struct { + name string + args []string + event string + }{ + {name: "restore", args: []string{"restore", "42"}, event: "restore runtime"}, + {name: "verify", args: []string{"verify", "system"}, event: "verify runtime"}, + {name: "gc dry run", args: []string{"gc", "--dry-run"}, event: "gc dry-run runtime"}, + {name: "gc live", args: []string{"gc"}, event: "gc live runtime"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + operationErr := fmt.Errorf("%s failed", test.event) + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add(test.event) + defer trace.add(test.event + " cleanup") + return operationErr + } + + _, stderr, code := captureRuntimeCLI(t, test.args, runtime) + if code != exitGeneral { + t.Fatalf("exit=%d want=%d stderr=%q", code, exitGeneral, stderr) + } + if !strings.Contains(stderr, operationErr.Error()) { + t.Fatalf("stderr=%q does not contain operation failure", stderr) + } + trace.require(t, []string{ + "lease acquire", + "recovery", + test.event, + test.event + " cleanup", + "lease release", + }) + }) + } +} + +func TestExecuteCLIVerifyOperationAndReleaseFailuresRemainJoined(t *testing.T) { + verifyErr := errors.New("verify runtime failure") + releaseErr := errors.New("verify lease release failure") + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return &fakeCLILease{releaseFn: func() error { + trace.add("lease release") + return releaseErr + }}, nil + }, + } + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("verify runtime") + defer trace.add("verify runtime cleanup") + return verifyErr + } + parsed := parsedCommandLine{method: "verify", positionals: []string{"system"}, flags: map[string][]string{}} + + err := executeCLICommand( + []string{"verify", "system"}, + parsed, + outputModeText, + repositoryCoordinationPolicyFor(parsed), + runtime, + ) + if !errors.Is(err, verifyErr) || !errors.Is(err, releaseErr) { + t.Fatalf("joined error=%v does not preserve verify and release failures", err) + } + trace.require(t, []string{ + "lease acquire", + "recovery", + "verify runtime", + "verify runtime cleanup", + "lease release", + }) +} diff --git a/cmd/coldkeep/repository_coordination_runtime_linux_test.go b/cmd/coldkeep/repository_coordination_runtime_linux_test.go new file mode 100644 index 00000000..46fd921f --- /dev/null +++ b/cmd/coldkeep/repository_coordination_runtime_linux_test.go @@ -0,0 +1,160 @@ +//go:build linux + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/coordination" + "github.com/franchoy/coldkeep/internal/observability" + "github.com/franchoy/coldkeep/internal/recovery" +) + +func TestRunCLIRealCoordinatorWrapsRestoreVerifyAndGCOnLinux(t *testing.T) { + tests := []struct { + name string + args []string + operation coordination.Operation + }{ + {name: "restore", args: []string{"restore", "42"}, operation: coordination.OperationRestore}, + {name: "verify", args: []string{"verify", "system"}, operation: coordination.OperationVerify}, + {name: "gc dry run", args: []string{"gc", "--dry-run"}, operation: coordination.OperationGarbageCollect}, + {name: "gc live", args: []string{"gc"}, operation: coordination.OperationGarbageCollect}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repositoryPath := t.TempDir() + identity, err := coordination.ResolveIdentity(repositoryPath) + if err != nil { + t.Fatalf("ResolveIdentity: %v", err) + } + ownerPath := filepath.Join(repositoryPath, coordination.ControlDirectoryName, coordination.OwnerMetadataName) + lockPath := filepath.Join(repositoryPath, coordination.ControlDirectoryName, coordination.LockArtifactName) + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = coordination.NewCoordinator + runtime.resolveIdentity = func(string) (coordination.Identity, error) { return identity, nil } + runtime.recover = func(cliOutputMode) (recovery.Report, error) { + trace.add("recovery") + return recovery.Report{}, nil + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("command runtime") + defer trace.add("command cleanup") + data, readErr := os.ReadFile(ownerPath) + if readErr != nil { + return readErr + } + owner, decodeErr := coordination.DecodeOwner(data) + if decodeErr != nil { + return decodeErr + } + if owner.Operation != test.operation { + return fmt.Errorf("owner operation=%q want=%q", owner.Operation, test.operation) + } + return nil + } + + _, stderr, code := captureRuntimeCLI(t, test.args, runtime) + if code != exitSuccess { + t.Fatalf("runCLIWithRuntime code=%d stderr=%q", code, stderr) + } + trace.require(t, []string{"recovery", "command runtime", "command cleanup", "render success"}) + if info, statErr := os.Lstat(lockPath); statErr != nil { + t.Fatalf("persistent repository.lock missing: %v", statErr) + } else if !info.Mode().IsRegular() { + t.Fatalf("repository.lock mode=%v want regular", info.Mode()) + } + if _, statErr := os.Lstat(ownerPath); !os.IsNotExist(statErr) { + t.Fatalf("owner metadata exists after release, stat err=%v", statErr) + } + + lease, acquireErr := coordination.NewCoordinator().Acquire(context.Background(), identity, mustTestOwnerRequest(t, identity, test.operation)) + if acquireErr != nil { + t.Fatalf("reacquire after command cleanup: %v", acquireErr) + } + if releaseErr := lease.Release(); releaseErr != nil { + t.Fatalf("release reacquired lease: %v", releaseErr) + } + }) + } +} + +func mustTestOwnerRequest(t *testing.T, identity coordination.Identity, operation coordination.Operation) coordination.Request { + t.Helper() + owner, err := coordination.NewOwner(operation, identity, "test", time.Unix(1_700_000_000, 0)) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + return coordination.Request{Operation: operation, Mode: coordination.ModeExclusive, Owner: owner} +} + +func TestRunCLIProductionLeaseWrapsRecoveryAndCommandOnLinux(t *testing.T) { + repositoryPath := t.TempDir() + controlDirectory := filepath.Join(repositoryPath, coordination.ControlDirectoryName) + lockPath := filepath.Join(controlDirectory, coordination.LockArtifactName) + ownerPath := filepath.Join(controlDirectory, coordination.OwnerMetadataName) + + originalContainersDir := container.ContainersDir + originalRecovery := startupRecoveryPhase + originalStats := runObservabilityStatsPhase + t.Cleanup(func() { + container.ContainersDir = originalContainersDir + startupRecoveryPhase = originalRecovery + runObservabilityStatsPhase = originalStats + }) + container.ContainersDir = repositoryPath + + assertPublishedOwner := func(stage string) { + t.Helper() + data, err := os.ReadFile(ownerPath) + if err != nil { + t.Fatalf("%s: read owner metadata: %v", stage, err) + } + owner, err := coordination.DecodeOwner(data) + if err != nil { + t.Fatalf("%s: decode owner metadata: %v", stage, err) + } + if owner.Operation != coordination.OperationStats { + t.Fatalf("%s: owner operation=%q want=%q", stage, owner.Operation, coordination.OperationStats) + } + } + startupRecoveryPhase = func(string) (recovery.Report, error) { + assertPublishedOwner("recovery") + return recovery.Report{}, nil + } + runObservabilityStatsPhase = func(observability.StatsOptions) (*observability.StatsResult, error) { + assertPublishedOwner("command") + return &observability.StatsResult{}, nil + } + + stdout, stderr, code := captureProductionCLI(t, []string{"stats"}) + if code != exitSuccess { + t.Fatalf("runCLI code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + if info, err := os.Lstat(lockPath); err != nil { + t.Fatalf("persistent repository.lock missing: %v", err) + } else if !info.Mode().IsRegular() { + t.Fatalf("repository.lock mode=%v want regular", info.Mode()) + } + if _, err := os.Lstat(ownerPath); !os.IsNotExist(err) { + t.Fatalf("owner metadata exists after runCLI, stat err=%v", err) + } +} + +func captureProductionCLI(t *testing.T, args []string) (stdout string, stderr string, code int) { + t.Helper() + stderr = captureStderr(t, func() { + stdout = captureStdout(t, func() { + code = runCLI(args) + }) + }) + return stdout, stderr, code +} diff --git a/cmd/coldkeep/repository_coordination_runtime_test.go b/cmd/coldkeep/repository_coordination_runtime_test.go new file mode 100644 index 00000000..663e48b0 --- /dev/null +++ b/cmd/coldkeep/repository_coordination_runtime_test.go @@ -0,0 +1,609 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/batch" + "github.com/franchoy/coldkeep/internal/coordination" + "github.com/franchoy/coldkeep/internal/recovery" +) + +func TestRunCLIRepositoryLeaseOrdering(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.dispatch = func(parsed parsedCommandLine, mode cliOutputMode) error { + trace.add("db open") + defer trace.add("db cleanup") + trace.add("command") + fmt.Fprintln(os.Stdout, "coordinated success") + return nil + } + + stdout, stderr, code := captureRuntimeCLI(t, []string{"stats"}, runtime) + if code != exitSuccess { + t.Fatalf("runCLIWithRuntime code=%d stderr=%q", code, stderr) + } + if !strings.Contains(stdout, "coordinated success") { + t.Fatalf("stdout=%q want coordinated success", stdout) + } + trace.require(t, []string{ + "lease acquire", "recovery", "db open", "command", "db cleanup", "lease release", "render success", + }) +} + +func TestExecuteCLILeaseAcquisitionFailureShortCircuitsRuntime(t *testing.T) { + busyErr := fmt.Errorf("%w: held by another process", coordination.ErrRepositoryBusy) + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + spoolDirectory := t.TempDir() + runtime.newOutputSpool = testOutputSpoolFactory(spoolDirectory) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return nil, busyErr + }, + } + } + parsed := parsedCommandLine{method: "stats", flags: map[string][]string{}} + + err := executeCLICommand( + []string{"stats"}, + parsed, + outputModeText, + repositoryCoordinationPolicyFor(parsed), + runtime, + ) + if !errors.Is(err, coordination.ErrRepositoryBusy) { + t.Fatalf("executeCLICommand error=%v want ErrRepositoryBusy", err) + } + trace.require(t, []string{"lease acquire"}) + requireDirectoryEmpty(t, spoolDirectory) +} + +func TestExecuteCLISpoolCreationFailureShortCircuitsBeforeLease(t *testing.T) { + spoolErr := errors.New("spool creation failure") + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newOutputSpool = func() (*coordinatedOutputSpool, error) { + return nil, spoolErr + } + parsed := parsedCommandLine{method: "stats", flags: map[string][]string{}} + + err := executeCLICommand( + []string{"stats"}, + parsed, + outputModeText, + repositoryCoordinationPolicyFor(parsed), + runtime, + ) + if !errors.Is(err, spoolErr) { + t.Fatalf("executeCLICommand error=%v want errors.Is(%v)", err, spoolErr) + } + trace.require(t, nil) +} + +func TestRunCLIRepositoryCoordinationBypassesNonRepositoryPaths(t *testing.T) { + tests := []struct { + name string + args []string + wantCode int + want []string + }{ + {name: "help", args: []string{"help"}, wantCode: exitSuccess, want: []string{"dispatch", "render success"}}, + {name: "version", args: []string{"version"}, wantCode: exitSuccess, want: []string{"dispatch", "render success"}}, + {name: "init", args: []string{"init"}, wantCode: exitSuccess, want: []string{"dispatch", "render success"}}, + {name: "simulate", args: []string{"simulate", "gc"}, wantCode: exitSuccess, want: []string{"dispatch", "render success"}}, + {name: "benchmark", args: []string{"benchmark", "run"}, wantCode: exitSuccess, want: []string{"dispatch", "render success"}}, + {name: "unknown command", args: []string{"unknown"}, wantCode: exitUsage, want: []string{"dispatch"}}, + {name: "command help", args: []string{"store", "--help"}, wantCode: exitSuccess, want: []string{"dispatch", "render success"}}, + {name: "invalid syntax", args: []string{"stats", "--output"}, wantCode: exitUsage}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + if test.name == "unknown command" { + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("dispatch") + return usageErrorf("unknown command") + } + } + stdout, stderr, code := captureRuntimeCLI(t, test.args, runtime) + _ = stdout + _ = stderr + if code != test.wantCode { + t.Fatalf("runCLIWithRuntime code=%d want=%d", code, test.wantCode) + } + trace.require(t, test.want) + }) + } +} + +func TestExecuteCLIRecoveryFailureRemainsDiagnosticAndReleasesLease(t *testing.T) { + recoveryErr := errors.New("startup recovery failure") + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.recover = func(cliOutputMode) (recovery.Report, error) { + trace.add("recovery") + return recovery.Report{}, recoveryErr + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("db open") + trace.add("command") + return nil + } + parsed := parsedCommandLine{method: "stats", flags: map[string][]string{}} + + err := executeCLICommand( + []string{"stats"}, + parsed, + outputModeText, + repositoryCoordinationPolicyFor(parsed), + runtime, + ) + if err != nil { + t.Fatalf("diagnostic startup recovery error became fatal: %v", err) + } + trace.require(t, []string{"lease acquire", "recovery", "db open", "command", "lease release"}) +} + +func TestExecuteCLIRuntimeFailuresReleaseLeaseAndPreserveErrors(t *testing.T) { + dbErr := errors.New("database open failure") + commandErr := errors.New("command failure") + releaseErr := errors.New("lease release failure") + + tests := []struct { + name string + operation error + release error + wantErrors []error + }{ + {name: "database failure", operation: dbErr, wantErrors: []error{dbErr}}, + {name: "command failure", operation: commandErr, wantErrors: []error{commandErr}}, + {name: "release failure", release: releaseErr, wantErrors: []error{releaseErr}}, + { + name: "operation and release failures", + operation: commandErr, + release: releaseErr, + wantErrors: []error{commandErr, releaseErr}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return &fakeCLILease{releaseFn: func() error { + trace.add("lease release") + return test.release + }}, nil + }, + } + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("db open") + defer trace.add("db cleanup") + trace.add("command") + return test.operation + } + parsed := parsedCommandLine{method: "stats", flags: map[string][]string{}} + + err := executeCLICommand( + []string{"stats"}, + parsed, + outputModeText, + repositoryCoordinationPolicyFor(parsed), + runtime, + ) + for _, wantErr := range test.wantErrors { + if !errors.Is(err, wantErr) { + t.Fatalf("executeCLICommand error=%v want errors.Is(%v)", err, wantErr) + } + } + trace.require(t, []string{ + "lease acquire", "recovery", "db open", "command", "db cleanup", "lease release", + }) + }) + } +} + +func TestRunCLIReleaseFailureSuppressesSpooledSuccessAndSuccessRendering(t *testing.T) { + releaseErr := errors.New("lease release failure") + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return &fakeCLILease{releaseFn: func() error { + trace.add("lease release") + return releaseErr + }}, nil + }, + } + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("command") + fmt.Fprintln(os.Stdout, "must not be rendered") + return nil + } + + stdout, _, code := captureRuntimeCLI(t, []string{"stats"}, runtime) + if code != exitGeneral { + t.Fatalf("runCLIWithRuntime code=%d want=%d", code, exitGeneral) + } + if strings.TrimSpace(stdout) != "" { + t.Fatalf("stdout=%q want empty after release failure", stdout) + } + trace.require(t, []string{"lease acquire", "recovery", "command", "lease release"}) +} + +func TestRunCLICoordinatedOutputDecisionMatrix(t *testing.T) { + operationErr := errors.New("operation failure") + releaseErr := errors.New("lease release failure") + + tests := []struct { + name string + operationErr error + releaseErr error + wantCode int + wantPayload bool + wantSuccess bool + wantStderrText []string + }{ + { + name: "operation success release success", + wantCode: exitSuccess, + wantPayload: true, + wantSuccess: true, + }, + { + name: "operation success release failure", + releaseErr: releaseErr, + wantCode: exitGeneral, + wantStderrText: []string{"repository coordination failed"}, + }, + { + name: "operation failure release success", + operationErr: operationErr, + wantCode: exitGeneral, + wantPayload: true, + wantStderrText: []string{operationErr.Error()}, + }, + { + name: "operation failure release failure", + operationErr: operationErr, + releaseErr: releaseErr, + wantCode: exitGeneral, + wantPayload: true, + wantStderrText: []string{operationErr.Error()}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + spoolDirectory := t.TempDir() + runtime.newOutputSpool = testOutputSpoolFactory(spoolDirectory) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return &fakeCLILease{releaseFn: func() error { + trace.add("lease release") + return test.releaseErr + }}, nil + }, + } + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + trace.add("command") + fmt.Fprintln(os.Stdout, "command payload") + return test.operationErr + } + + stdout, stderr, code := captureRuntimeCLI(t, []string{"stats"}, runtime) + if code != test.wantCode { + t.Fatalf("runCLIWithRuntime code=%d want=%d stderr=%q", code, test.wantCode, stderr) + } + if got := strings.Contains(stdout, "command payload"); got != test.wantPayload { + t.Fatalf("payload present=%v want=%v stdout=%q", got, test.wantPayload, stdout) + } + for _, want := range test.wantStderrText { + if !strings.Contains(stderr, want) { + t.Fatalf("stderr=%q does not contain %q", stderr, want) + } + } + trace.mu.Lock() + successRendered := false + for _, event := range trace.events { + if event == "render success" { + successRendered = true + } + } + trace.mu.Unlock() + if successRendered != test.wantSuccess { + t.Fatalf("success rendered=%v want=%v", successRendered, test.wantSuccess) + } + requireDirectoryEmpty(t, spoolDirectory) + }) + } +} + +func TestRunCLICoordinatedBatchPartialFailureOutput(t *testing.T) { + tests := []struct { + name string + args []string + outputMode cliOutputMode + }{ + {name: "human", args: []string{"repair", "ref-counts"}, outputMode: outputModeText}, + {name: "json", args: []string{"repair", "ref-counts", "--output", "json"}, outputMode: outputModeJSON}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.dispatch = func(parsed parsedCommandLine, mode cliOutputMode) error { + if mode != test.outputMode { + t.Fatalf("output mode=%q want=%q", mode, test.outputMode) + } + return emitBatchCommandReport("repair", partialFailureBatchReport(), mode) + } + + stdout, stderr, code := captureRuntimeCLI(t, test.args, runtime) + if code == exitSuccess { + t.Fatalf("runCLIWithRuntime unexpectedly succeeded stdout=%q stderr=%q", stdout, stderr) + } + if test.outputMode == outputModeJSON { + var payload map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &payload); err != nil { + t.Fatalf("decode batch JSON stdout=%q: %v", stdout, err) + } + if got, _ := payload["status"].(string); got != "partial_failure" { + t.Fatalf("batch JSON status=%q want=partial_failure payload=%v", got, payload) + } + } else { + for _, want := range []string{"[REPAIR]", "✔ id=12", "✖ id=18", "Summary:"} { + if !strings.Contains(stdout, want) { + t.Fatalf("human batch stdout=%q does not contain %q", stdout, want) + } + } + } + }) + } +} + +func TestExecuteCLIBatchOperationAndReleaseFailuresReplayReportAndPreserveErrors(t *testing.T) { + releaseErr := errors.New("lease release failure") + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + return &fakeCLILease{releaseFn: func() error { return releaseErr }}, nil + }, + } + } + var operationErr error + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + operationErr = emitBatchCommandReport("repair", partialFailureBatchReport(), outputModeText) + return operationErr + } + parsed := parsedCommandLine{method: "repair", positionals: []string{"ref-counts"}, flags: map[string][]string{}} + + var executeErr error + stdout := captureStdout(t, func() { + executeErr = executeCLICommand( + []string{"repair", "ref-counts"}, + parsed, + outputModeText, + repositoryCoordinationPolicyFor(parsed), + runtime, + ) + }) + if !strings.Contains(stdout, "Summary:") { + t.Fatalf("batch report was not replayed, stdout=%q", stdout) + } + if operationErr == nil || !errors.Is(executeErr, operationErr) { + t.Fatalf("execute error=%v does not preserve operation error=%v", executeErr, operationErr) + } + if !errors.Is(executeErr, releaseErr) { + t.Fatalf("execute error=%v does not preserve release error=%v", executeErr, releaseErr) + } +} + +func TestRunCLIReleaseOnlyFailureSuppressesJSONSuccessPayload(t *testing.T) { + releaseErr := errors.New("lease release failure") + trace := &cliLifecycleTrace{} + runtime := newTestCLIRuntime(t, trace) + runtime.newCoordinator = func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + return &fakeCLILease{releaseFn: func() error { return releaseErr }}, nil + }, + } + } + runtime.dispatch = func(parsedCommandLine, cliOutputMode) error { + fmt.Fprintln(os.Stdout, `{"status":"ok","command":"stats"}`) + return nil + } + + stdout, stderr, code := captureRuntimeCLI(t, []string{"stats", "--output", "json"}, runtime) + if code != exitGeneral { + t.Fatalf("runCLIWithRuntime code=%d want=%d stderr=%q", code, exitGeneral, stderr) + } + if strings.TrimSpace(stdout) != "" { + t.Fatalf("stdout=%q want empty after release-only failure", stdout) + } + if !strings.Contains(stderr, "repository coordination failed") { + t.Fatalf("stderr=%q does not contain stable coordination error", stderr) + } +} + +func partialFailureBatchReport() batch.Report { + return batch.NewReport(batch.OperationRepair, false, []batch.ItemResult{ + {ID: 12, Status: batch.ResultSuccess, Message: "repaired"}, + {ID: 18, Status: batch.ResultFailed, Message: "forced failure"}, + }) +} + +func newTestCLIRuntime(t *testing.T, trace *cliLifecycleTrace) cliRuntime { + t.Helper() + identity, err := coordination.ResolveIdentity(t.TempDir()) + if err != nil { + t.Fatalf("ResolveIdentity: %v", err) + } + return cliRuntime{ + newCoordinator: func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + trace.add("lease acquire") + return &fakeCLILease{releaseFn: func() error { + trace.add("lease release") + return nil + }}, nil + }, + } + }, + newOutputSpool: testOutputSpoolFactory(t.TempDir()), + resolveIdentity: func(string) (coordination.Identity, error) { + return identity, nil + }, + newOwner: coordination.NewOwner, + recover: func(cliOutputMode) (recovery.Report, error) { + trace.add("recovery") + return recovery.Report{}, nil + }, + dispatch: func(parsedCommandLine, cliOutputMode) error { + trace.add("dispatch") + return nil + }, + renderSuccess: func(parsedCommandLine, cliOutputMode) { + trace.add("render success") + }, + now: func() time.Time { return time.Unix(1_700_000_000, 0) }, + } +} + +func newDefaultCommandTestRuntime(t *testing.T) cliRuntime { + t.Helper() + identity, err := coordination.ResolveIdentity(t.TempDir()) + if err != nil { + t.Fatalf("ResolveIdentity: %v", err) + } + return cliRuntime{ + newCoordinator: func() coordination.Coordinator { + return &fakeCLICoordinator{ + acquireFn: func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) { + return &fakeCLILease{releaseFn: func() error { return nil }}, nil + }, + } + }, + newOutputSpool: testOutputSpoolFactory(t.TempDir()), + resolveIdentity: func(string) (coordination.Identity, error) { return identity, nil }, + newOwner: coordination.NewOwner, + recover: runStartupRecoveryWithOptionalLogBuffering, + dispatch: dispatchCLICommand, + renderSuccess: printCLISuccess, + now: time.Now, + } +} + +func testOutputSpoolFactory(directory string) func() (*coordinatedOutputSpool, error) { + return func() (*coordinatedOutputSpool, error) { + return newCoordinatedOutputSpool(directory) + } +} + +func requireDirectoryEmpty(t *testing.T, directory string) { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("read spool directory: %v", err) + } + if len(entries) != 0 { + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + paths = append(paths, filepath.Join(directory, entry.Name())) + } + t.Fatalf("spool directory not empty after lifecycle: %v", paths) + } +} + +func captureRuntimeCLI(t *testing.T, args []string, runtime cliRuntime) (stdout string, stderr string, code int) { + t.Helper() + stderr = captureStderr(t, func() { + stdout = captureStdout(t, func() { + code = runCLIWithRuntime(args, runtime) + }) + }) + return stdout, stderr, code +} + +type fakeCLICoordinator struct { + acquireFn func(context.Context, coordination.Identity, coordination.Request) (coordination.Lease, error) +} + +func (coordinator *fakeCLICoordinator) Acquire( + ctx context.Context, + identity coordination.Identity, + request coordination.Request, +) (coordination.Lease, error) { + return coordinator.acquireFn(ctx, identity, request) +} + +type fakeCLILease struct { + releaseOnce sync.Once + releaseFn func() error + releaseErr error +} + +func (lease *fakeCLILease) Release() error { + lease.releaseOnce.Do(func() { + lease.releaseErr = lease.releaseFn() + }) + return lease.releaseErr +} + +type cliLifecycleTrace struct { + mu sync.Mutex + events []string +} + +func (trace *cliLifecycleTrace) add(event string) { + trace.mu.Lock() + defer trace.mu.Unlock() + trace.events = append(trace.events, event) +} + +func (trace *cliLifecycleTrace) require(t *testing.T, want []string) { + t.Helper() + trace.mu.Lock() + got := append([]string(nil), trace.events...) + trace.mu.Unlock() + if len(got) != len(want) { + t.Fatalf("trace=%v want=%v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("trace=%v want=%v", got, want) + } + } +} diff --git a/cmd/coldkeep/snapshot_create_engine_routing_test.go b/cmd/coldkeep/snapshot_create_engine_routing_test.go index 369d28cd..1f56e60e 100644 --- a/cmd/coldkeep/snapshot_create_engine_routing_test.go +++ b/cmd/coldkeep/snapshot_create_engine_routing_test.go @@ -7,6 +7,7 @@ import ( "database/sql/driver" "encoding/json" "errors" + "fmt" "strings" "sync/atomic" "testing" @@ -26,6 +27,8 @@ type trackingQueryDriver struct { queryCount *int32 } +var trackingQueryDriverSequence uint64 + type trackingQueryConn struct { queryCount *int32 } @@ -54,9 +57,7 @@ func (c trackingQueryConn) QueryContext(context.Context, string, []driver.NamedV func openTrackingQueryDB(t *testing.T, queryCount *int32) *sql.DB { t.Helper() - driverName := "snapshot-create-routing-driver-" + strings.ReplaceAll(t.Name(), "/", "-") - sql.Register(driverName, trackingQueryDriver{queryCount: queryCount}) - dbconn, err := sql.Open(driverName, "") + dbconn, err := newTrackingQueryDB(queryCount) if err != nil { t.Fatalf("sql.Open tracking driver: %v", err) } @@ -64,6 +65,63 @@ func openTrackingQueryDB(t *testing.T, queryCount *int32) *sql.DB { return dbconn } +func newTrackingQueryDB(queryCount *int32) (*sql.DB, error) { + // database/sql registrations are process-global. The driver retains the + // caller's counter pointer, so every fixture needs its own driver instance. + driverName := fmt.Sprintf( + "snapshot-create-routing-driver-%d", + atomic.AddUint64(&trackingQueryDriverSequence, 1), + ) + sql.Register(driverName, trackingQueryDriver{queryCount: queryCount}) + return sql.Open(driverName, "") +} + +func TestTrackingQueryDBFixtureIsRepeatableAndIsolated(t *testing.T) { + var firstCount, secondCount int32 + first := openTrackingQueryDB(t, &firstCount) + _ = openTrackingQueryDB(t, &secondCount) + + if _, err := first.QueryContext(context.Background(), "SELECT legacy_count"); err == nil { + t.Fatal("expected tracking driver query to fail") + } + if got := atomic.LoadInt32(&firstCount); got != 1 { + t.Fatalf("expected first fixture to observe one query, got %d", got) + } + if got := atomic.LoadInt32(&secondCount); got != 0 { + t.Fatalf("expected second fixture to remain isolated, got %d queries", got) + } +} + +func TestTrackingQueryDBFixtureParallelCreation(t *testing.T) { + const fixtureCount = 16 + errs := make(chan error, fixtureCount) + for range fixtureCount { + go func() { + var queryCount int32 + dbconn, err := newTrackingQueryDB(&queryCount) + if err != nil { + errs <- fmt.Errorf("create fixture: %w", err) + return + } + defer func() { _ = dbconn.Close() }() + if _, err := dbconn.QueryContext(context.Background(), "SELECT legacy_count"); err == nil { + errs <- errors.New("expected tracking driver query to fail") + return + } + if got := atomic.LoadInt32(&queryCount); got != 1 { + errs <- fmt.Errorf("expected isolated fixture query count 1, got %d", got) + return + } + errs <- nil + }() + } + for range fixtureCount { + if err := <-errs; err != nil { + t.Fatal(err) + } + } +} + func TestRunSnapshotCommandCreateOmitsIDForEngineGeneration(t *testing.T) { originalLoad := loadDefaultStorageContextPhase originalCreate := createSnapshotPhase diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 25485822..c53b2f73 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -25,6 +25,220 @@ evaluation in v1.7. ## Running benchmarks +### v1.13.11 release-gate policy + +The final Phase 11 benchmark governance policy is documented in +`docs/release/v1.13/v1.13.11-phase11-benchmark-governance-policy.md`. +Required CI treats functional evidence as hard and GitHub-hosted timing as +advisory: + +- `benchmark-integrity` runs the four bounded v2 profiles with exactly two + candidate-only samples and no warmup. Schema, fixture, execution, semantic + state, counters, cleanup, confidentiality, inventory, and checksums are hard. +- `benchmark-timing-advisory` retains all four historical `small` observations. + Valid threshold crossings are explicit `BENCHMARK_TIMING_WARNING` results, + never hard performance failures or passes. The ordinary `small` path does + not capture `diagnostic_final_state`; that hard semantic authority belongs + only to `benchmark-integrity`. +- `CI Required Gate` requires both families to complete successfully. Missing, + malformed, unverified, skipped, or failed evidence remains blocking. + +Hard performance enforcement is `deferred_to_controlled_infrastructure`. +Historical v1.9 inputs are `historical_v1.9_absolute` advisory material only. +No paired required job, production paired mode, paired manifest, or numeric +paired threshold policy is active. + +The historical command's successful completion establishes that its temporary +benchmark database cleanup completed. Explicit per-case and repeat cleanup +evidence, semantic final state, and repeat hard-state equality remain integrity +responsibilities. Candidate/configuration/I/O/contract failures produce a +machine-readable `BENCHMARK_TIMING_EVALUATION_FAILURE` report and exit 2 after +evidence finalization; they are never converted to an advisory warning or +`BENCHMARK_TIMING_NOT_EVALUATED`. + +### Retained paired diagnostic tooling + +The same-run paired contract in +`docs/release/v1.13/v1.13.11-phase11-paired-benchmark-gate-contract.md` remains +non-production diagnostic/controlled-runner tooling. +The rejected oversized candidates `ci-paired-w1-v1` and `ci-paired-w4-v1` +remain immutable historical inputs. The bounded diagnostic candidates +`ci-paired-w1-v2` and `ci-paired-w4-v2` execute the same complete ordered +nine-case schema-v2 suite with per-case isolation. `scripts/paired_benchmark_gate.py` +owns the fixed `C R` warmups, fixed five-pair production or ten-pair diagnostic +inventory, paired ratio/MAD comparison, hard-state equivalence, checksums, and +four-profile decision aggregation. Diagnostic aggregation requires the trusted +caller to select its authority explicitly: + +```bash +python3 scripts/paired_benchmark_gate.py decision \ + --mode diagnostic \ + --profile none-w1=/controlled/none-w1 \ + --profile none-w4=/controlled/none-w4 \ + --profile zstd-w1=/controlled/zstd-w1 \ + --profile zstd-w4=/controlled/zstd-w4 \ + --output-dir /controlled/decision +``` + +The exact `/controlled/decision` child must not exist before invocation. The +caller may create `/controlled`, verifies the child is absent, and then lets +the harness create and exclusively own the child. The same rule applies to +every `sample --output-dir`: workflow-owned parent, nonexistent harness-owned +child, and artifact upload from that exact child after the command returns. +Pre-creating an empty child is an evidence-integrity failure; deleting and +recreating a populated child does not repair ownership. + +A successful diagnostic sample and four-profile decision are classified +`DIAGNOSTIC_QUALIFIED`, never production `PASS`. The decision reconstructs +statistics and correctness evidence from checksummed raw reports, requires +byte-identical reference/candidate binaries, and records diagnostic-only scope, +authority, and sampler-owned per-profile elapsed time. + +No production reference manifest or numeric paired threshold is authorized. +The harness contract requires production sampling and production decisions to +remain hard-disabled even if ungoverned files appear; enabling them requires a +later authorized governance and trusted-base integration change. Diagnostic +mode qualifies the architecture only and cannot consume production artifacts; +production decisions cannot consume diagnostic artifacts. Diagnostic sampling +and aggregation are implemented. Binary-identical run `30696834430` accepted +the bounded fixtures and functional evidence lifecycle but rejected +GitHub-hosted paired timing as a 5% hard endpoint. A future +separately authorized launcher will use this command shape: + +```bash +profile_parent=/controlled/profile-parent +profile_output="${profile_parent}/artifact" +mkdir -p "${profile_parent}" +test ! -e "${profile_output}" + +COLDKEEP_CODEC=aes-gcm python3 scripts/paired_benchmark_gate.py sample \ + --reference-binary /controlled/reference/coldkeep \ + --candidate-binary /controlled/candidate/coldkeep \ + --reference-sha <40-character-sha> \ + --candidate-sha <40-character-sha> \ + --output-dir "${profile_output}" \ + --dataset ci-paired-w4-v2 \ + --compression none \ + --workers 4 \ + --mode diagnostic \ + --pairs 10 \ + --go-version '' \ + --postgres-version '' \ + --database-image-digest 'sha256:' +``` + +The command timeout is fixed at 600 seconds and is capped by the remaining +35-minute sampler-owned profile budget. The harness terminates and reaps the +active process group when that budget expires, performs compensating cleanup, +and emits a checksummed non-authoritative +`DIAGNOSTIC_TIME_BUDGET_EXCEEDED` artifact. Qualification requires all ten +pairs, median ratios within 0.95–1.05, paired MAD no greater than 2.5%, exact +hard state, valid counters and cleanup, no timeout, and completion within the +35-minute diagnostic profile limit. A failure decision is also checksummed and +records missing, invalid, failed, verified, and not-evaluated profiles without +claiming absent evidence was verified. Exact bounds pass. Samples cannot be +discarded or extended. No paired reference manifest or numeric production +threshold policy exists. + +The v2 fixtures retain seed 1701, 1 KiB small files, the 1–256 KiB mixed range, +`remove_every=4`, all nine cases, and ten diagnostic pairs. Both use a 64 MiB +large file and 400 small files; w1 retains 400 mixed files and w4 retains 800. +The earlier v1 definitions remain parseable for historical evidence but are no +longer selected by the four-profile diagnostic matrix. + +Any future temporary launcher must keep sensitive values out of YAML `env:` +blocks, mask runner roots before checkout, provision its isolated container +only after masking, generate credentials and fixture material inside a +tracing-disabled shell, and upload already-finalized evidence with +`if: always()`. `scripts/audit_ci_enforcement.sh --paired-launcher ` +checks this source contract without adding or authorizing a workflow. Static +GitHub-managed container aliases such as `/github/workspace`, +`/github/runner_temp`, `/github/home`, and `/github/workflow` are allowed +runtime metadata; they are not credentials or benchmark-generated paths. +Actual host paths, dynamically generated roots and evidence paths, database +and container identifiers, namespaces, credentials, ports, DSNs, and key +material remain confidential and must be masked before possible output. + +The first bounded-v2 launcher attempt, run `30693345495`, failed before any +benchmark invocation because it created each exact sample output child before +calling the harness. It produced no fixture, paired-statistic, hard-state, +counter, cleanup, or time-budget result. Its decision command correctly +preserved a checksummed failure-shaped decision after the four profile +artifacts were absent. The corrected run `30696834430` later completed all four +profiles with valid checksums, equal hard state, valid counters, and complete +cleanup. Its timing variation rejected hosted performance authority. No +additional remote performance diagnostic is required by the final policy. + +The stopped local A/A probe completed only its two excluded warmups and is +non-authoritative, non-PASS partial evidence. It is not qualification evidence +and made no reference or threshold decision. The completed four-profile remote +ten-pair A/A run is diagnostic evidence only. + +The legacy `ci-stable-v1` material below remains historical diagnostic +compatibility and has no paired performance authority. + +The superseded `ci-stable-v1` proposal used fixed larger fixtures and one +isolated PostgreSQL database per case. It still requires `--repeat 1` and can +be captured for historical diagnostics by: + +```bash +python3 scripts/benchmark_gate.py sample \ + --binary ./coldkeep \ + --output-dir /tmp/coldkeep-gate-none-w4 \ + --compression none \ + --workers 4 \ + --warmups 1 \ + --samples 5 \ + --postgres-version "PostgreSQL 16.14" \ + --database-image-digest "sha256:" +``` + +The legacy sampler rejects malformed, repeated, trailing, incomplete, +reordered, or fixture-inconsistent reports and preserves every raw sample. Its +absolute duration median is not a Phase 11 performance endpoint. + +#### Outcome E evidence contract + +Preserved calibration run `30176935742` showed stable workers=1 counters and +scheduling-sensitive workers=4 container open, close, and fsync counts. A +bounded local final-state diagnostic then showed exact fixture, logical-file, +ordered chunk-graph, restored-tree, snapshot-membership, GC, verification, and +canonical physical-content evidence. Only execution allocation and physical +layout observations varied. This is Outcome E: the former all-counters-equal +calibration rule was over-constrained; it is not evidence of a product, +fixture, isolation, or aggregation correctness defect. + +Raw benchmark schema remains version 2. Every corrected-contract raw report +must contain `diagnostic_final_state` schema version 2. Reports that omit that +object, including the preserved GitHub workers=4 artifacts, remain useful +historical diagnostics but cannot enter paired comparison evidence. +Diagnostic-final-state schema-v1 evidence is historical only. Unknown fields in +validated sections fail closed until a schema and policy update classifies +them. + +Evidence policy version 2 assigns fields by semantics: + +| Policy | Fields and validation | +| --- | --- | +| `hard_equal` | Raw and aggregate schema/kind/status; capture source/binary and hard environment identity; codec, compression, dataset, workers, pipeline depth, and deterministic mode; every fixture constant and ordered case seed; warmup/sample counts; per-case processed file/byte totals; operation success/failure/skipped totals; expected restored totals; logical/status, ordered chunk-graph, restored-tree, snapshot, GC, verification, and placement-independent physical-content fingerprints/totals; cleanup success and zero leaked databases, processes, or temporary resources. | +| `derived_equal` | Throughput; aggregate execution totals; median, mean, min/max, sample standard deviation, MAD, MAD ratio, CV, sample order/count relationships, command p95, duplicated outer/I/O counters, snapshot-write sums, open/close balance, and applicable artifact/manifest hashes. Each value is recomputed from its source fields. | +| `bounded_nonnegative` | Mandatory signed-64-bit nonnegative per-case container opens, appends, fsyncs, bytes written, bytes read, and container closes. Values remain present and retained; append/read/write contradictions and open/close imbalance fail. No unsupported percentage bound is imposed. Raw-v2 zero-valued snapshot-write fields are omitted by the established schema, normalized to zero, retained operationally, and checked through their operation-specific and aggregate-sum relationships. | +| `informational` | Raw timings, retained operational samples and distributions, container/block allocation counts, container bytes, physical-layout digest, host load, free disk, hosted-runner image warning, and command timing distributions. These remain well-formed and sanitized but are not exact across samples. | +| `excluded_sensitive` | Credentials, passwords, encryption keys, DSNs, usernames, database names, repository or temporary paths, sensitive command arguments, environment dumps, and raw internal identifiers. Names and values are rejected before report acceptance. | + +The canonical physical-content digest is hard evidence. It includes logical +chunk identity, payload size, codec/compression transforms, and unreferenced +payload identity without database IDs, container IDs, or placement. Payload +bytes and chunk-reference totals are also hard. Container/block allocation and +the separate layout digest are informational only after every semantic and +canonical physical-content field matches. + +`revalidate-raw` applies the policy to preserved diagnostic samples and writes +a separately identified revalidation report. It retains every operational +sample and distribution and always records +`performance_calibration_status: not_evaluated`; it does not create a baseline +or claim calibration acceptance. + Phase 8 benchmark execution is script-only for v1.8 release hardening. The `coldkeep benchmark` command is available in the shipped CLI for ad-hoc @@ -135,19 +349,18 @@ JSON output exposes both per-case worker usage and an aggregate } ``` -## Current baseline +## Historical v1.9 baseline -The repository now maintains an official v1.9 baseline set for the -recommended packed production family (`aes-gcm` encryption): +The repository retains the v1.9 baseline set for historical interpretation of +the recommended packed production family (`aes-gcm` encryption): - compression modes: `none`, `zstd` - worker profiles: `w1`, `w4` - contract shape: `none/zstd × w1/w4` (four baseline JSON artifacts total) -These artifacts are now the frozen performance reference point for v1.10+ -architectural work. Future releases may reorganize benchmark runners or CI -gates, but they must compare against this frozen set unless an explicit -baseline-refresh decision is documented. +These artifacts preserve earlier release decisions. They have no performance +authority for the v1.13.11 paired gate, cannot be reused as paired samples, and +their absolute thresholds cannot be reinterpreted as ratio thresholds. Official v1.9 baseline files: @@ -270,18 +483,17 @@ retained for historical v1.6/v1.7 context. an identical `relative-path → digest` map across isolated runs, proving that user-visible restore output is byte-for-bit stable. -## Regression Thresholds (v1.9) +## Historical Regression Thresholds (v1.9) Benchmarks are now actionable through defined regression thresholds. Thresholds are mode-specific (uncompressed vs. compressed) and case-specific, balancing detection sensitivity with normal run-to-run variance. -**Official policy:** See [benchmarks/v1.9/regression-thresholds.yaml](../benchmarks/v1.9/regression-thresholds.yaml) -for the authoritative threshold definition. +**Historical policy:** See [benchmarks/v1.9/regression-thresholds.yaml](../benchmarks/v1.9/regression-thresholds.yaml) +for the frozen v1.9 threshold definition. -These thresholds are frozen for the v1.9 baseline set and are the reference -policy for v1.10+ regression detection until an explicit threshold-refresh -decision is approved. +These thresholds are frozen with the v1.9 baseline set. They do not define +paired-regression sensitivity. ### Uncompressed mode (packed + aes-gcm + none) @@ -294,7 +506,8 @@ decision is approved. | Metadata operation regression | > 3% | snapshot-creation, gc-after-churn, stats-inspect | | Memory increase | > 10% | Not yet enforced via CLI but monitored | -Any regression exceeding these thresholds **fails CI** and must be investigated or reverted. +Under the historical gate, a regression exceeding these thresholds failed CI +and required investigation. ### Compressed mode (packed + aes-gcm + zstd) @@ -713,16 +926,23 @@ Key assertions include: - ✓ **New blocks coexist safely:** new compressed packed blocks store/restore/verify alongside old legacy data - ✓ **Migration only additive:** old metadata path remains intact while new packed metadata is added for new chunks -## CI policy +## Historical absolute timing advisory -CI now separates correctness checks from benchmark measurement: +Required CI preserves the old observations without granting them hard timing +authority: -1. The correctness matrix runs independently from benchmarks and covers the supported codec combinations. -2. The benchmark matrix runs the small dataset only for the recommended packed `aes-gcm` production modes with `COLDKEEP_COMPRESSION=none` and `COLDKEEP_COMPRESSION=zstd`. -3. Benchmark outputs are captured as artifacts for inspection. Threshold-based regression comparison is enforced via `--compare` with mode-specific thresholds; violations are reported per the v1.9 regression thresholds policy above. +1. The correctness matrix and `benchmark-integrity` jobs enforce functional + contracts independently from timing. +2. `benchmark-timing-advisory` runs the `small` dataset for all `none/zstd × + w1/w4` packed `aes-gcm` profiles. +3. The candidate raw envelope and frozen baseline shape are validated before + comparison. Threshold crossings are recorded as + `BENCHMARK_TIMING_WARNING`; comparator or evidence defects still fail CI. +4. Artifacts preserve the raw observation, advisory report, input hashes, + violations, and exhaustive checksums. See [benchmarks/v1.9/regression-thresholds.yaml](../benchmarks/v1.9/regression-thresholds.yaml) -and CI workflow for authoritative threshold application. +and the CI workflow for historical implementation detail. ## Phase 4 implementation order diff --git a/docs/internal/benchmark_baselines_v1_9.md b/docs/internal/benchmark_baselines_v1_9.md index 65849a3f..99c27777 100644 --- a/docs/internal/benchmark_baselines_v1_9.md +++ b/docs/internal/benchmark_baselines_v1_9.md @@ -1,16 +1,27 @@ # Benchmark Baselines v1.9 (Internal) -Status: Frozen +Status: Frozen historical single-observation advisory evidence Date: 2026-05-09 -Scope: Official benchmark reference point for post-v1.9 work +Scope: Historical/informational evidence; no Phase 11 performance authority ## Purpose -This document freezes the official v1.9 benchmark baselines and the policy used -to compare future results against them. +This document freezes the historical v1.9 benchmark artifacts and records the +policy that originally compared later results against them. -The goal is to ensure that v1.10+ architectural work measures regressions -against one stable reference point instead of silently moving the floor. +The Phase 11 benchmark-gate investigation found that these files do not contain +distribution or resolved-environment provenance and that the active and +worker-specific manifests contain stale hashes. Their measurement content and +3%/5% thresholds remain frozen historical policy, but they must not be treated +as schema-v2 aggregate evidence or reinterpreted as paired-ratio thresholds. +Required CI may read them only as `historical_v1.9_absolute` advisory inputs +after validating their established legacy shape. A threshold crossing is +`BENCHMARK_TIMING_WARNING`, not a hard performance failure or pass. + +The artifacts continue to document earlier decisions. They do not select a +Phase 11 production reference or hard performance endpoint. Missing or malformed +baseline evidence remains an advisory-evaluation integrity error and fails the +job; advisory authority does not weaken evidence validation. These baselines are the reference for: @@ -39,7 +50,7 @@ Machine-readable manifests: - `benchmarks/v1.9/baselines/baseline-manifest-v1.9-small-w1-r1.json` (frozen `w1` profile) - `benchmarks/v1.9/baselines/baseline-manifest-v1.9-small-w4-r1.json` (frozen `w4` profile) -The authoritative regression-threshold policy is: +The historical v1.9 regression-threshold policy is: - `benchmarks/v1.9/regression-thresholds.yaml` @@ -76,9 +87,11 @@ Each baseline manifest must continue to prove: - same logical totals - same case set -## Regression Threshold Contract +## Historical Regression Threshold Contract -Thresholds are frozen in `benchmarks/v1.9/regression-thresholds.yaml`. +Thresholds are frozen in `benchmarks/v1.9/regression-thresholds.yaml` for +historical advisory interpretation only. They have no paired-gate or hard +performance authority. Policy summary: @@ -108,9 +121,9 @@ Requirements: ## Freeze Statement -These v1.9 baseline artifacts are the official reference point for future -regression detection until an explicit baseline-refresh decision supersedes -them. - -Architecture may change in v1.10+, but benchmark comparison authority remains -anchored to this v1.9 baseline set. \ No newline at end of file +These v1.9 artifacts remain immutable historical evidence. Required CI cites +them only under the hosted timing advisory policy and preserves their content, +manifests, hashes, and thresholds byte-for-byte. They cannot be reused as paired +samples. Future hard performance authority requires separately authorized +controlled infrastructure, qualification, reference governance, and numeric +threshold policy. diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index a37f60e7..61b69759 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -71,11 +71,89 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree and release/tag CI passed. - `release/v1.13.9` was deleted locally and remotely. No Phase 25 was required, and no mandatory v1.x runtime remediation remained. -- `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` is ready for - release after completing all eight phases and the local pre-release gate. -- `v1.13.10-release-train-reconciliation.md` is the canonical disposition of - retired historical v1.13.10–v1.13.13 allocations. -- `v1.13.10-engine-contract-documentation-truthfulness.md` records the active +- `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` is released and + operationally closed. It is a valid closure-integrity and CI-runtime-hygiene + baseline, not the final v1.x release. +- A post-release roadmap-to-code audit superseded the narrower final-v1.x + conclusion and restored v1.13.11–v1.13.13 for remaining must-before-v2 work. +- `v1.13.11 — Safety and Backend Compatibility Gate Closure` is the single + release ready for pre-release validation. Phases 0–20 are complete. Phase 12 implements the + repository-wide exclusive fail-fast Lease and proves native runtime plus the + production Coordinator lifecycle on Linux, macOS, and Windows. Phase 13 + preserves strengthened G6 integrity coverage and proves deterministic Linux + independent-process contention, killed-holder release with immediate + reacquisition, real live-GC cross-process exclusion, and dedicated + PostgreSQL advisory-session ownership. Its platform and same-host proof + limits are recorded in + [v1.13.11-phase13-closure.md](v1.13.11-phase13-closure.md). Phase 14 now + validates outer ranges before allocation/read, enforces supported-header, + catalog-maximum, and physical-size consistency, and preserves persisted + maxima across packed reads and recovery without changing format bytes. Its + evidence is recorded in + [v1.13.11-phase14-container-range-header-consistency.md](v1.13.11-phase14-container-range-header-consistency.md). + Phase 15 now validates exact decompression expectations before allocation, + caps complete CKBL output and zstd decoder memory/window resources at 4 MiB, + and applies the same intrinsic contract to identity and zstd codecs. Restore, + system Verify, and Store semantic reuse inherit the shared bounded path. Its + evidence is recorded in + [v1.13.11-phase15-bounded-decompression.md](v1.13.11-phase15-bounded-decompression.md). + Phase 16 now preserves exact integer tokens throughout the stable v1.7 + stats, inspect, and simulate-GC JSON envelope path without changing JSON + number types, schemas, APIs, storage, coordination, or error behavior. Its + evidence is recorded in + [v1.13.11-phase16-json-integer-fidelity.md](v1.13.11-phase16-json-integer-fidelity.md). + Phase 17 now inventories all 70 non-DDL production mutations and hardens the + 20 required-row gaps without changing valid zero-row cleanup, recovery, CAS, + upsert, bulk, or GC semantics. SQLite/PostgreSQL parity and rollback are + proven, including no physical GC deletion after a missed metadata delete. + Its evidence is recorded in + [v1.13.11-phase17-fail-closed-sql-mutations.md](v1.13.11-phase17-fail-closed-sql-mutations.md). + Phase 18 now makes selected existing PostgreSQL mutation-cardinality, + storage/recovery, and Linux coordination execution proof fail closed in + required CI without changing product semantics or job topology. Its evidence + is recorded in + [v1.13.11-phase18-required-backend-coordination-ci.md](v1.13.11-phase18-required-backend-coordination-ci.md). + Phase 19 reconciles the validation matrix, backend claim matrix, reusable + release checklist, active trackers, Phase 18 closure chronology, proof + boundaries, deferred items, and evidence links. Its documentation-only + evidence is recorded in + [v1.13.11-phase19-validation-evidence-reconciliation.md](v1.13.11-phase19-validation-evidence-reconciliation.md). + Phase 20 freezes the prospective exact-head gate contract and pre-release + state. The immutable commit containing that record must pass candidate-head + CI, Required Gate, CodeQL, and the complete clean local Profile A gate before + one pull request to `main` is authorized. BKC-016 remains + `Deferred — documented`. Phase 12 closure remains recorded in + [v1.13.11-phase12-closure.md](v1.13.11-phase12-closure.md). The diagnostic + benchmark-gate bootstrap remains recorded in + [v1.13.11-phase11-benchmark-gate-integrity-remediation.md](v1.13.11-phase11-benchmark-gate-integrity-remediation.md). + Phase 10 implementation `ad82c959` passed exact-head CI run `30148670910`, + including all five PostgreSQL events in plain job `89655223183` and required + gate `89656972706`. A first benchmark variance was resolved by successful + same-head rerun `89656813012` without any benchmark accommodation. BKC-003 + and BKC-015 are backend-specific — proven within their documented bounds. + Phase 9 exact-head + CI run `30114444798` at `848e579b` proved scoped active, uncontended Engine + mutation and GC dry-run parity, including all five required PostgreSQL events + in plain job `89551564893` and required-gate job `89555865026`. Phase 8 exact-head CI run `30109561344` at `bcae3576` proved + its scoped snapshot selector and tree-presentation contracts, including both + required PostgreSQL selector events and the aggregate required gate. Phase 7 exact-head CI run `29993172886` at `313d0069` + proved the scoped engine read-side contracts across SQLite and PostgreSQL, + including the SQLite deep-verification single-connection correction. Phase 6 + exact-head CI run `29983479388` proved the scoped + implemented catalog contracts across SQLite and PostgreSQL, including + deterministic snapshot ordering. Selected schema/bootstrap/migration + contracts and the G6 fail-closed remediation also have exact-head CI + evidence, while broad backend parity remains intentionally unclaimed. + v1.13.11 is ready for exact-head pre-release validation; the latest released + version remains v1.13.10. Merge, tag, and publication remain later + operations. Its canonical trackers are + `v1.13.11-phase0-post-release-closure-correction-and-baseline.md`, + `v1.13.11-scope.md`, `v1.13.11-phase-list.md`, + `v1.13.11-validation-checklist.md`, and `v1.13.11-release-gate.md`. +- The updated `v1.13.x-release-train.md` is the authoritative current plan; + final v1.x completion is gated by v1.13.11–v1.13.13. v2.0 implementation has + not started. +- `v1.13.10-engine-contract-documentation-truthfulness.md` records the current Engine contract boundary and its intentional limitations. - `v1.13.10-release-state-validator-contract.md` freezes the lifecycle, evidence, parsing, CKRS rule, output, fixture, and CI integration contract. @@ -85,10 +163,5 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree upload-artifact v7 migration and semantic-preservation evidence. - `v1.13.10-v1x-closure-summary-and-v2.0-handoff-freeze.md` freezes the final v1.x baseline, explicit v2.0 inputs, and v2/v3 scope boundary. -- The release gate records the initial benchmark blocker, its bounded - benchmark-infrastructure correction, the later package-interaction test - isolation correction, and a fresh passed Profile A gate on `53b66dda`. - The evidence-restoration candidate remains subject to its required clean - exact-head gate before one pull request from `release/v1.13.10` to `main` is - authorized. External PR, merge, tag, and publication evidence remains - unavailable. +- v1.13.10's public release, tag, merge, tag-CI, and deleted-release-branch + evidence is recorded separately from its historical pre-release gate narrative. diff --git a/docs/release/v1.13/v1.13.10-phase-list.md b/docs/release/v1.13/v1.13.10-phase-list.md index dc7e329a..2ba665ac 100644 --- a/docs/release/v1.13/v1.13.10-phase-list.md +++ b/docs/release/v1.13/v1.13.10-phase-list.md @@ -1,8 +1,14 @@ # Coldkeep v1.13.10 Phase List **Release:** `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` -**Status:** Ready for release -**Branch:** `release/v1.13.10` +**Status:** Released and operationally closed +**Released branch:** `release/v1.13.10` (historical) + +> **Current-status notice (post-release correction):** This phase list is an +> accurate historical record of the v1.13.10 tagged-release decision. Its +> final-v1.x conclusion was superseded after release by a roadmap-to-code audit +> that restored substantive v1.13.11–v1.13.13 work. See the authoritative +> `v1.13.x-release-train.md` and v1.13.11 Phase 0 artifact. ## Phase 0 — v1.13.9 Post-Release Documentation Closure and v1.13.10 Baseline diff --git a/docs/release/v1.13/v1.13.10-release-gate.md b/docs/release/v1.13/v1.13.10-release-gate.md index ce171e0f..3ea01ec0 100644 --- a/docs/release/v1.13/v1.13.10-release-gate.md +++ b/docs/release/v1.13/v1.13.10-release-gate.md @@ -1,17 +1,32 @@ # Coldkeep v1.13.10 Release Gate **Release:** `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` -**Status:** Passed — awaiting publication +**Status:** Passed and released ## Release identity - Version: `v1.13.10` - Title: `v1.x Closure Integrity and CI Runtime Hygiene` -- Branch: `release/v1.13.10` +- Released branch: `release/v1.13.10` (historical) - Execution date: `2026-07-18` in Europe/Madrid - Source version: `1.13.10` - Lifecycle state: `pre-release` +> **Current-status notice (post-release correction):** This gate accurately +> describes the v1.13.10 tagged-release decision. Its final-v1.x conclusion +> was superseded after release by a roadmap-to-code audit that restored +> substantive v1.13.11–v1.13.13 work. The updated release train and v1.13.11 +> Phase 0 artifact are authoritative current planning sources. + +## Post-release provenance + +Public GitHub evidence confirms stable release `Coldkeep v1.13.10 — v1.x +Closure Integrity and CI Runtime Hygiene`, published July 19, 2026 at 18:01. +Its tag targets `423c57815580c39bee4f79ecd81570e9cfa9d273`, the merge of PR +#105. Tag-triggered CI run #502 succeeded with 19 jobs in 18m26s, and +`release/v1.13.10` is absent from the public branch list. The local GitHub CLI +token was invalid; public GitHub pages supplied the independent evidence. + The exact final release candidate is the commit containing this corrected release-gate evidence. Its complete exact-head validation is reported by Git and the release pull request evidence. This document does not guess or embed @@ -34,7 +49,7 @@ backend-default change was introduced by Phase 8. Phases 0 through 8 are complete. The active scope, phase list, validation checklist, release train, root README, release README, and changelog agree that -v1.13.10 is ready for release. No phase remains `Next` or `Not started`. A +Historical pre-release statement: v1.13.10 is ready for release. No phase remains `Next` or `Not started`. A complete Profile A gate passed on `53b66dda`; the evidence-restoration candidate must complete the entire gate again on its clean exact head before the single release pull request is operationally authorized. @@ -153,9 +168,11 @@ productization. LAN/NAS, centralized server mode, multi-user authorization, cloud/object storage, distributed repositories, replication, and cross-machine coordination remain v3.x or later. -## External post-PR and post-tag evidence +## Historical pre-release external-evidence list -The following evidence does not yet exist and is not claimed locally: +At the time of the original pre-release narrative, the following evidence did +not yet exist. This historical list is superseded by the confirmed public +post-release provenance above: - pull-request CI, CodeQL, or Codacy result; - pull-request merge evidence or main-branch validation; diff --git a/docs/release/v1.13/v1.13.10-release-train-reconciliation.md b/docs/release/v1.13/v1.13.10-release-train-reconciliation.md index ebbec00c..c6f0dac9 100644 --- a/docs/release/v1.13/v1.13.10-release-train-reconciliation.md +++ b/docs/release/v1.13/v1.13.10-release-train-reconciliation.md @@ -1,5 +1,12 @@ # Coldkeep v1.13.10 Phase 2 — v1.13.x Release-Train Reconciliation +> **Current-status notice (post-release correction):** This artifact accurately +> records the v1.13.10 tagged-release decision and the narrower conclusion then +> made. That final-v1.x conclusion was superseded after release by a +> roadmap-to-code audit that restored substantive v1.13.11–v1.13.13 work. The +> authoritative current planning sources are `v1.13.x-release-train.md` and +> `v1.13.11-phase0-post-release-closure-correction-and-baseline.md`. + ## Purpose This document reconciles release numbering and scope disposition. It does not diff --git a/docs/release/v1.13/v1.13.10-scope.md b/docs/release/v1.13/v1.13.10-scope.md index 7b62e7b2..d7dc5f18 100644 --- a/docs/release/v1.13/v1.13.10-scope.md +++ b/docs/release/v1.13/v1.13.10-scope.md @@ -1,9 +1,16 @@ # Coldkeep v1.13.10 Scope **Release:** `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` -**Status:** Ready for release -**Branch:** `release/v1.13.10` -**Starting baseline:** `09f0f7824df14e3f1be2c3d791325eba7a8adef9` +**Status:** Released and operationally closed +**Released branch:** `release/v1.13.10` (historical) +**Release target:** `423c57815580c39bee4f79ecd81570e9cfa9d273` + +> **Current-status notice (post-release correction):** This tracker accurately +> describes the v1.13.10 tagged-release decision. v1.13.10 remains a valid +> released closure-integrity and CI-runtime-hygiene baseline, but its +> final-v1.x conclusion was superseded after release by the roadmap-to-code +> audit. The authoritative current planning sources are +> `v1.13.x-release-train.md` and the v1.13.11 Phase 0 artifact. ## Goal @@ -38,7 +45,20 @@ v2.0 handoff posture. compatibility remains protected. 4. v2.0 implementation has not started. -## Release workflow +## Post-release evidence + +Public GitHub evidence confirms: + +- Stable release: `Coldkeep v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene`. +- Published: July 19, 2026 at 18:01. +- Tag target: `423c57815580c39bee4f79ecd81570e9cfa9d273`; merge provenance: PR #105. +- Tag-triggered CI: run #502, successful, 19 jobs, 18m26s. +- `release/v1.13.10` is absent from the public branch list. + +The local GitHub CLI could not retrieve this evidence because its available +token was invalid; public GitHub pages were used as the independent source. + +## Historical release workflow All phases remain on `release/v1.13.10`. The exact-name test-isolation remediation `53b66dda` completed a fresh Profile A gate after the benchmark @@ -49,11 +69,8 @@ operations. ## Release-train reconciliation -Phase 2 records disposition rather than claiming complete implementation -absorption. Retired former release numbers remain historically traceable; -incomplete backend-proof and coupling work remains explicit v2.x scope. This -phase does not authorize deferred implementation, and the v1.13.9 handoff -review identified no mandatory v1.x runtime remediation. +This is the historical pre-release workflow. The post-release audit instead +restored v1.13.11–v1.13.13 as must-before-v2 work; v1.13.11 is now active. ## Phase summary diff --git a/docs/release/v1.13/v1.13.10-v1x-closure-summary-and-v2.0-handoff-freeze.md b/docs/release/v1.13/v1.13.10-v1x-closure-summary-and-v2.0-handoff-freeze.md index 2384df9b..ec5bb6e0 100644 --- a/docs/release/v1.13/v1.13.10-v1x-closure-summary-and-v2.0-handoff-freeze.md +++ b/docs/release/v1.13/v1.13.10-v1x-closure-summary-and-v2.0-handoff-freeze.md @@ -1,5 +1,12 @@ # Coldkeep v1.13.10 Phase 7 — v1.x Closure Summary and v2.0 Handoff Freeze +> **Current-status notice (post-release correction):** This artifact accurately +> describes the v1.13.10 tagged-release decision and its then-current final-v1.x +> conclusion. That conclusion was superseded after release by a roadmap-to-code +> audit that restored substantive v1.13.11–v1.13.13 work. The authoritative +> current planning sources are `v1.13.x-release-train.md` and the v1.13.11 +> Phase 0 artifact; v2.0 implementation remains not started. + ## Purpose Freeze the checked-in v1.x baseline from which v2.0 planning may begin. This diff --git a/docs/release/v1.13/v1.13.10-validation-checklist.md b/docs/release/v1.13/v1.13.10-validation-checklist.md index f1caf140..6ac1f097 100644 --- a/docs/release/v1.13/v1.13.10-validation-checklist.md +++ b/docs/release/v1.13/v1.13.10-validation-checklist.md @@ -1,8 +1,14 @@ # Coldkeep v1.13.10 Validation Checklist **Release:** `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` -**Status:** Ready for release -**Branch:** `release/v1.13.10` +**Status:** Released and operationally closed +**Released branch:** `release/v1.13.10` (historical) + +> **Current-status notice (post-release correction):** This checklist +> accurately records the v1.13.10 tagged-release decision. Its final-v1.x +> conclusion was superseded after release by the roadmap-to-code audit that +> restored substantive v1.13.11–v1.13.13 work. Current planning is defined by +> `v1.13.x-release-train.md` and the v1.13.11 Phase 0 artifact. ## Global release controls diff --git a/docs/release/v1.13/v1.13.11-backend-compatibility-claim-matrix.md b/docs/release/v1.13/v1.13.11-backend-compatibility-claim-matrix.md new file mode 100644 index 00000000..df84ab57 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-backend-compatibility-claim-matrix.md @@ -0,0 +1,90 @@ +# Coldkeep v1.13.11 Backend Compatibility Claim Matrix + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** `2 — Backend Compatibility Claim Matrix` +**Status:** Complete +**Branch:** `release/v1.13.11` +**Evidence basis:** Phase 2 source inspection at `40a9462a`; execution updates +through Phase 10 exact-head CI run `30148670910` at `ad82c959`; Phase 11 +closure at accepted exact head `b08da99a8efe39b5c9cdb0bea59362304bd2fc09`; +Phase 12–13 coordination closure; and Phase 18 required execution proof at +implementation head `eaa5896` and closure head `9c1fa524`. + +## Executive finding + +PostgreSQL is the normal CLI/runtime backend and is exercised by required +integration, smoke, benchmark, and adversarial CI. SQLite has substantial +unit/package coverage and targeted repository-portability evidence. That is +mostly separate evidence, not same-contract parity proof. + +The catalog contract suite is the current concrete same-contract exception. +Phase 6 executed comparable SQLite/PostgreSQL fixtures and assertions in the +required plain correctness-matrix package-contract step. `quality` still has +no PostgreSQL service or `COLDKEEP_TEST_DB`, so its package execution remains +SQLite-only. + +## Required-CI execution map + +| CI area | Backend evidence actually executed | Compatibility meaning | +| --- | --- | --- | +| `quality` | Package tests for both codecs; SQLite is available; no PostgreSQL service or `COLDKEEP_TEST_DB`. | PostgreSQL-gated internal tests, including catalog contract subtests, skip. | +| `correctness-matrix`, `legacy-compatibility`, `integration-stress`, `integration-long-run` | PostgreSQL service plus `COLDKEEP_TEST_DB=1`; integration evidence. | PostgreSQL behavior evidence, not reusable SQLite/PostgreSQL parity proof. | +| `adversarial`, `smoke` | PostgreSQL service and CLI/runtime evidence; Phase 18 requires selected Linux process events in adversarial JSON. | PostgreSQL safety evidence, not SQLite parity or cross-platform subprocess proof. | +| `benchmark-integrity`, `benchmark-timing-advisory` | PostgreSQL service and four-profile benchmark evidence. | Hard candidate integrity plus informational hosted timing, not backend parity or hard timing-regression authority. | +| `cross-platform` | Selected path, storage restore, engine restore, and snapshot restore tests. | Platform evidence; not a backend-parity job. | +| `ci-required` | Aggregates quality, correctness, stress, long-run, adversarial, smoke, legacy, benchmark-integrity, benchmark-timing-advisory, and cross-platform jobs. | Includes selected named PostgreSQL/process proof; it does not turn unrelated separate evidence into parity. | + +Phase 4's required PostgreSQL internal-package step now executes selected +harness-backed contracts against PostgreSQL. `quality` continues to supply the +separate SQLite package execution; this split does not by itself prove broad +backend parity. + +## Canonical claims + +| Claim ID | Domain/layer | Operation or contract | Current claim and source | SQLite implementation path / evidence | PostgreSQL implementation path / evidence | Required-CI execution status | Classification | Compatibility meaning | Identified evidence gap and risk if overstated | Owning later phase / proposed executable proof | Notes or deliberate exclusions | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| BKC-001 | Runtime identity | Normal CLI backend | `internal/db/db.go`; release scope. | Simulated/test and portability contexts only. | `ConnectDB`; required integration, smoke, benchmark, and adversarial jobs. | PostgreSQL runtime paths execute. | Backend-specific — proven | Normal CLI runtime is PostgreSQL; SQLite is not a normal runtime mode. | Calling SQLite a supported CLI default would be false. | Current boundary; v2.x for SQLite-first product direction. | PostgreSQL compatibility remains required. | +| BKC-002 | Engine construction | Caller-provided `*sql.DB` | `engine.Config` and `backend_compat_test.go`. | `TestEngineNewAcceptsBackendNeutralDB` uses `:memory:`. | No corresponding construction assertion. | SQLite only in `quality`. | Unproven | Public shape is neutral, but operational construction parity is not proved. | PostgreSQL construction can be assumed without an executable assertion. | Later Phase 3 adoption; parameterize construction fixture for both backends. | Harness infrastructure exists; construction proof has not started. | +| BKC-003 | Database layer | Detection and optional lock-query selection | `internal/db/backend.go` and Phase 10 TXN-001/004–007. | SQLite detection, capability routing, clause omission, and executable base queries passed. | PostgreSQL supported clauses and server-backed contracts passed. | Five exact `/postgres` events passed in run `30148670910`, plain job `89655223183`; required gate `89656972706` passed. | Backend-specific — proven | Backend detection and tested clause routing execute as documented; SQL remains backend-specific. | This is not identical isolation or general parity. | Phase 10 complete; retain selectors. | PostgreSQL emits/executes supported clauses; SQLite intentionally omits them. | +| BKC-004 | Schema | SQLite fresh bootstrap and migrations | `RunMigrations`, `migrations*_test.go`, and Phase 5 SCH fixtures. | Embedded SQLite schema, migrations, legacy fixtures, SCH-001–008, and local execution. | Required PostgreSQL SCH-001–007 execution is recorded in run `29803865860`, job `88550222280`; PostgreSQL historical starts differ. | SQLite package contracts and selected PostgreSQL SCH events passed in required CI. | Separate evidence — no parity proof | SQLite fresh bootstrap and migration behavior has strong executable evidence. This proves the SQLite contract only; selected shared bootstrap outcomes do not establish SQLite/PostgreSQL migration parity. | Treating SQLite success, or selected fresh-schema results, as historical migration parity would overstate evidence. | Phase 5 closed; extend only precisely scoped schema contracts later. | SQLite v12 preservation remains SQLite-specific. | +| BKC-005 | Schema | PostgreSQL bootstrap, version detection, auto-migration | `EnsurePostgresSchema`, PostgreSQL migration tests, and Phase 5 SCH-009. | Separate SQLite migration fixtures. | Embedded `schema_postgres.sql`, gated tests, and PostgreSQL SCH-001–007/009. | Required PostgreSQL selector pass events recorded in run `29803865860`, job `88550222280`. | Separate evidence — no parity proof | PostgreSQL bootstrap, version detection, selected current-schema behavior, and v11 auto-migration have executable evidence. Different historical fixtures prevent a general parity conclusion. | PostgreSQL and SQLite historical paths are not comparable shared fixtures. | Phase 5 closed; preserve asymmetric historical paths unless a later contract proves an exact shared outcome. | Schema version remains 16. | +| BKC-006 | Schema | FK, uniqueness, ordering, and dialect SQL | `db/schema_sqlite.sql`, `db/schema_postgres.sql`, and Phase 5 SCH-005–007. | SQLite behavioral invariant tests. | Matching PostgreSQL SCH-005–007 tests executed in required CI. | Required PostgreSQL selector pass events recorded in run `29803865860`, job `88550222280`. | Unproven | Selected uniqueness, foreign-key, nullable/default, and version-metadata outcomes passed on both backends; no general schema equality, ordering, or dialect-SQL guarantee is established. | Constraint divergence can still produce different outcomes outside the listed invariants. | Phase 5 closed; classify any future invariant only after its own comparable proof. | SQL mechanisms need not match. | +| BKC-007 | Catalog | Find logical/physical files, snapshots, list, reachability | Phase 6 CAT-001–005 in `backend_contract_test.go`. | Shared fixture and comparable assertions execute in package tests. | The same fixture and assertions passed in CI run `29983479388`, plain correctness job `89130181273`. | Required CI recorded CAT-001–005 PostgreSQL pass events at `db12c3d2`. | Equivalent — proven | The five implemented methods have comparable SQLite/PostgreSQL fixture assertions and required-CI PostgreSQL execution. | This is not a claim about deferred APIs, engine reads, mutations, or unlisted catalog behavior. | Phase 6 complete; preserve the six selectors. | No production catalog change was needed beyond CAT-004 ordering. | +| BKC-008 | Catalog | Deferred graph, placements, restore-plan, and GC-plan APIs | CAT-006; `graph.go`, `placement.go`, `restore_plan.go`, `gc_plan.go`. | Explicit non-mutating `ErrNotImplemented` contract executes. | The same deferred boundary passed in CI run `29983479388`. | Required CI recorded CAT-006 PostgreSQL pass at `db12c3d2`. | Deferred — documented | These APIs remain explicitly unavailable, not parity guarantees. | Do not imply operational catalog support or an implementation. | v1.13.12; implement only under its catalog-completion scope. | No deferred API was activated. | +| BKC-009 | Catalog | Placeholders, nullable values, ordering, and catalog errors | CAT-002–004/007 in `backend_contract_test.go`; CAT-004 correction `db12c3d2`. | Shared fixture asserts nullable timestamps/booleans, filtering, limits, deterministic ordering, and cancelled contexts. | The same assertions passed in CI run `29983479388`, plain correctness job `89130181273`. | Required CI recorded all six catalog PostgreSQL pass events after CAT-004 correction. | Equivalent — proven | Placeholder binding, tested nullable values, `created_at DESC, id DESC` ordering, filtering, limits, and cancelled-context errors have comparable SQLite/PostgreSQL proof. | This does not promise wildcard escaping, case-insensitive search, raw database errors, or all catalog error classes. | Phase 6 complete; later rows own broader catalog/error behavior. | CAT-004 failure in run `29982838592` was corrected and exact-head verified. | +| BKC-010 | Engine read side | Stats, Inspect, Verify | Phase 7 ENG-R-001/003/004; implementation `3e8c98df`; deep-verification correction `313d0069`. | Shared fixture, the bounded single-connection deep-verification regression, and the successful quality job `89160468816` in run `29993172886`. | The comparable ENG-R-001/003/004 PostgreSQL contracts passed in plain correctness job `89160468856` in run `29993172886`. | Exact-head required CI recorded all four Phase 7 PostgreSQL selectors; quality and aggregate gate `89165096957` succeeded. | Equivalent — proven | Stats, Inspect, Verify, and their tested context, error, and non-mutation behavior have comparable SQLite/PostgreSQL fixture assertions and exact-head required-CI execution. | This does not cover mutation, GC, locking, selector resolution, or unlisted engine behavior. | Phase 7 complete; retain the four selectors. | Deep byte verification is unchanged; rows are released before nested block queries. | +| BKC-011 | Snapshot read side | List, show, stats, diff, selectors | Phase 7 ENG-R-002/004 plus Phase 8 `TestEngineSnapshotSelectorsAcrossBackends` and `TestEngineSnapshotSelectorErrorsAcrossBackends`. | Explicit-ID views and the shared selector/error contracts execute in the SQLite package path. | Phase 8 PostgreSQL selector and selector-error subtests passed in exact-head run `30109561344`, plain job `89535269535`. | Required CI recorded both Phase 8 `/postgres` events; quality job `89535269631` and aggregate gate `89540053230` also passed. | Equivalent — proven | Implemented list/show/stats/diff selectors, file-query filtering, invalid-regex errors, deterministic results, and CLI tree presentation have scoped SQLite/PostgreSQL proof. | This deliberately excludes mutations, restore selectors, latest/tags/batch IDs, label case behavior, wildcard escaping, locking, and unlisted read behavior. | Phase 8 complete; preserve the two selectors. | Do not confuse this scoped read proof with mutation or broad backend parity. | +| BKC-012 | Engine mutation | Store, Restore, Remove, snapshot create/delete/restore | Phase 9 MUT-001–006/008 in `mutation_backend_contract_test.go` plus existing package/integration/adversarial suites. | All four mutation contracts pass focused SQLite once, ten repeated runs, and the race profile with semantic repository/container fingerprints. | The same five shared PostgreSQL subtests passed in exact-head CI run `30114444798`, plain job `89551564893`. | All five Phase 9 `/postgres` events, quality, and required gate `89555865026` passed. | Equivalent — proven | Tested active, uncontended Store/Remove/Restore and snapshot mutation outcomes, stable results/errors, rollback boundaries, and repository/destination effects have comparable proof. | This is not a claim about recursive Store, CLI store-folder, dormant fields, mid-operation cancellation, concurrency/locks, live GC, or untested combinations. | Phase 9 complete; Phase 10 owns locking. | No production change was needed. | +| BKC-013 | GC | Dry-run | Phase 9 MUT-007/008 in `mutation_backend_contract_test.go` plus existing maintenance/integration evidence. | Fixed dead-container plan, reachability counts, repeated ordering, pre-cancel, and DB/container non-mutation pass SQLite and race profiles. | The shared PostgreSQL GC dry-run subtest passed in exact-head CI run `30114444798`, plain job `89551564893`. | `TestEngineGCDryRunAcrossBackends/postgres`, quality, and required gate `89555865026` passed. | Equivalent — proven | Tested GC dry-run plan/result ordering and non-mutation behavior have comparable proof. | Live GC, locking, contention, and byte reclamation planning remain outside this claim. | Phase 9 complete; BKC-014 retains live-GC ownership. | Live GC remains BKC-014 and is intentionally outside this contract. | +| BKC-014 | GC | Live GC | Phase 9 dry-run boundary plus Phase 13C live-GC and advisory-session evidence. | SQLite live GC remains explicitly rejected; dry-run remains supported. | PostgreSQL live GC holds one dedicated advisory-lock session and participates in the repository Lease. | Phase 13 implementation/evidence runs passed; Phase 18 requires the named live-GC and advisory-session events in existing required jobs. | Backend-specific — proven | SQLite refusal and PostgreSQL live execution are intentional backend-specific contracts. Linux real live-GC cross-process exclusion is proven. | Do not normalize the backend difference or infer cross-host exclusion from the PostgreSQL advisory lock. | Closed within the Phase 13 proof boundary; Phase 18 preserves named execution. | PostgreSQL advisory ownership is an inner singleton mechanism; it is not distributed repository coordination. | +| BKC-015 | Transactions | Row locks, isolation, `FOR UPDATE`, `NOWAIT`, `SKIP LOCKED`, advisory locks | Phase 10 TXN-002–009 contracts plus Phase 13C dedicated advisory-session correction. | Common transaction behavior and intentional clause omission passed; SQLite live GC remains unsupported. | Server-observed lock release, `55P03`, SKIP LOCKED ordering, cancellation, container integration, same-session advisory acquisition/unlock, cleanup, and single-connection refusal passed. | Five Phase 10 `/postgres` events passed in run `30148670910`; Phase 18 retains four advisory-session events and required PostgreSQL execution proof. | Backend-specific — proven | Common transaction outcomes are proven; PostgreSQL row/advisory locks and SQLite omissions remain separate bounded guarantees. | This is not identical isolation, SQLite row locking, broad backend parity, or deadlock/serializable retry proof. | Phase 10 row-lock and Phase 13 advisory-session work complete; preserve required selectors. | No broad backend equivalence claim. | +| BKC-016 | Coordination | Same-host locking and repository coordination | Phase 11 contract; Phase 12 native/Coordinator implementation; Phase 13 Linux process and live-GC closure; Phase 18 required named-event preservation. | Same-process reservation and production Coordinator lifecycle execute in the SQLite-backed package/runtime paths; live GC remains explicitly unsupported for SQLite. | PostgreSQL production Coordinator lifecycle, Linux process contention, killed-holder release, live-GC exclusion, and dedicated advisory-session ownership execute within their documented scopes. | Native runtime and Coordinator lifecycle are required on Linux, macOS, and Windows; Phase 18 requires Linux independent-process, killed-holder, and live-GC pass events with no matching skip. | Deferred — documented | Bounded same-host local-filesystem coordination is proven: native/Coordinator runtime on Linux/macOS/Windows and representative subprocess semantics on Linux. The classification remains deferred because broader cross-host/network-filesystem and separate macOS/Windows subprocess guarantees are not part of the release claim. | Claiming separate macOS/Windows subprocess execution, cross-host/distributed exclusion, or network-filesystem safety would overstate the evidence. | Phase 12–13 bounded proof complete; Phase 18 required-CI preservation complete. Broader coordination remains outside v1.13.11. | FreeBSD is compile-only/unsupported; owner metadata is diagnostic; BKC-016 remains `Deferred — documented`. | +| BKC-017 | Portability | SQLite-first repository and default-backend claim | Release roadmap; repository portability integration test. | SQLite catalog portability evidence. | PostgreSQL remains normal runtime and compatibility target. | SQLite portability test is not a runtime switch. | Deferred — documented | SQLite-first is future direction, not current default behavior. | Product/runtime support could be overstated. | v2.x. | No SQLite-default initialization in v1.13.11. | +| BKC-018 | Candidates | Repair and recovery | Engine candidate/deferred-boundary records. | No active backend guarantee. | No active backend guarantee. | None applicable. | Deferred — documented | Candidate-only work is not a backend-parity claim. | Unsupported paths could be represented as active. | v1.13.12 / v2.x. | Keep `ErrNotImplemented` distinctions explicit. | + +## Corrections and exclusions + +- The catalog fixture accurately records optional local PostgreSQL execution + and names the required-CI correctness-matrix package-contract step. Required + PostgreSQL execution is recorded; Phase 6 owns expanded catalog parity proof. +- “Dual-backend” catalog descriptions must distinguish required-CI execution + from complete catalog-parity coverage. +- PostgreSQL integration evidence plus SQLite unit evidence is classified as + separate evidence, never as parity. +- Phase 3 implements reusable test-fixture infrastructure only. It does not + activate PostgreSQL package tests, change backend behavior, or add locking. + +## Phase ownership + +Phases 3 and 4 completed fixture infrastructure and required PostgreSQL +package-contract activation. Phase 5 completed selected schema/bootstrap and +migration evidence while retaining conservative classifications. Phase 6 +completed the scoped implemented-catalog proof. Phase 7 completed scoped +engine read-side proof, Phase 8 closed the scoped selector proof, and Phase 9 +closed its scoped mutation and GC dry-run proof. Phase 10 completed bounded +backend-specific transaction and row-lock proof. Phases 11–13 completed the +bounded coordination contract, native implementation, Linux process proof, +live-GC barrier, and PostgreSQL advisory-session ownership. Phase 18 made the +selected backend and coordination execution proof fail closed in required CI. +Deferred catalog APIs belong to v1.13.12; SQLite-first product behavior belongs +to v2.x. diff --git a/docs/release/v1.13/v1.13.11-phase-list.md b/docs/release/v1.13/v1.13.11-phase-list.md new file mode 100644 index 00000000..327ea0c2 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase-list.md @@ -0,0 +1,275 @@ +# Coldkeep v1.13.11 Phase List + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Status:** Ready for release +**Branch:** `release/v1.13.11` + +## Phase 0 — v1.13.10 Post-Release Closure Correction and v1.13.11 Baseline + +**Status:** Complete + +## Phase 1 — v1.13.11 Release Identity Activation + +**Status:** Complete + +- Evidence artifact: [v1.13.11-phase1-release-identity-activation.md](v1.13.11-phase1-release-identity-activation.md). +- Updated source, version-test, and reusable checklist identity from `1.13.10` + to `1.13.11`. +- Verified human and JSON CLI version reporting and a zero-violation + release-state validator result. + +## Phase 2 — Backend Compatibility Claim Matrix + +**Status:** Complete + +- Evidence artifact: [v1.13.11-backend-compatibility-claim-matrix.md](v1.13.11-backend-compatibility-claim-matrix.md). +- Classified current SQLite/PostgreSQL claims, actual required-CI execution, + intentional differences, unproven behavior, and later-phase ownership. + +Phase 2 classified evidence; Phase 3 added only reusable fixture +infrastructure. No later parity proof is represented as implemented. + +## Phase 3 — Reusable Dual-Backend Test Harness + +**Status:** Complete + +- Evidence artifact: [v1.13.11-phase3-reusable-dual-backend-test-harness.md](v1.13.11-phase3-reusable-dual-backend-test-harness.md). +- Added isolated SQLite and optional PostgreSQL package-test fixtures and + adopted them for the existing catalog contract suite. +- Required-CI PostgreSQL activation remains intentionally unimplemented. + +## Phase 4 — Existing PostgreSQL-Gated Package Suite CI Activation + +**Status:** Complete + +- Evidence artifact: [v1.13.11-phase4-existing-postgresql-gated-package-suite-ci-activation.md](v1.13.11-phase4-existing-postgresql-gated-package-suite-ci-activation.md). +- Required CI run `29729981751` recorded the required PostgreSQL pass events. + +## Phase 5 — Schema, Bootstrap, and Migration Parity + +**Status:** Complete + +- Evidence artifact: [v1.13.11-phase5-schema-bootstrap-and-migration-parity.md](v1.13.11-phase5-schema-bootstrap-and-migration-parity.md). +- Schema-contract implementation: `2b603b7c`; diagnostic checkpoint: + `bfe49176`; fail-closed G6 correction: `54ecd84c`. +- CI run `29803865860` proved required PostgreSQL SCH execution but exposed + the independent G6 blocker. Final run `29815330238`, attempt 2, passed after + the single authorized same-SHA benchmark retry; required-gate job + `89121652949` succeeded. + +## Phase 6 — Implemented Catalog Contract Parity + +**Status:** Complete + +- Evidence artifact: [v1.13.11-phase6-implemented-catalog-contract-parity.md](v1.13.11-phase6-implemented-catalog-contract-parity.md). +- CAT-001–007 executed at correction `db12c3d2` in required CI run + `29983479388`; the plain correctness job recorded all six PostgreSQL + selector pass events and the aggregate required gate succeeded. + +## Phase 7 — Engine Read-Side Backend Parity + +**Status:** Complete + +- Evidence artifact: [v1.13.11-phase7-engine-read-side-backend-parity.md](v1.13.11-phase7-engine-read-side-backend-parity.md). +- ENG-R-001–004 passed exact-head CI run `29993172886` at `313d0069`, including + all four required PostgreSQL events, quality, and the aggregate required + gate. +- The SQLite deep-verification single-connection starvation correction is + covered by a bounded regression without altering byte verification. + +## Phase 8 — Snapshot Selector Determinism Closure + +**Status:** Complete + +- Implementation record: [v1.13.11-phase8-snapshot-selector-determinism-closure.md](v1.13.11-phase8-snapshot-selector-determinism-closure.md). +- Implementation `bcae3576` passed exact-head CI run `30109561344`, including + quality, both required PostgreSQL selector events in plain job `89535269535`, + and aggregate required-gate job `89540053230`. + +## Phase 9 — Engine Mutation Backend Parity + +**Status:** Complete + +- Implementation record: + [v1.13.11-phase9-engine-mutation-backend-parity.md](v1.13.11-phase9-engine-mutation-backend-parity.md). +- Implementation `848e579b` passed exact-head CI run `30114444798`, including + quality, all five required PostgreSQL mutation/GC events in plain job + `89551564893`, and aggregate required-gate job `89555865026`. +- BKC-012 and BKC-013 are equivalently proven only within the contracts' scoped + active, uncontended mutation and GC dry-run boundaries. + +## Phase 10 — Backend Transaction and Row-Lock Semantics + +**Status:** Complete + +- Implementation record: + [v1.13.11-phase10-backend-transaction-row-lock-semantics.md](v1.13.11-phase10-backend-transaction-row-lock-semantics.md). +- Five shared contracts cover the common transaction boundary, real + PostgreSQL row-lock behavior, SQLite clause omission, cancellation and + cleanup, and production container NOWAIT/SKIP LOCKED integration. +- Exact-head CI run `30148670910` at `ad82c959` passed all five PostgreSQL + events in plain job `89655223183` and required gate `89656972706`. +- No production code, public contract, schema, transaction setting, or retry + policy changed. + +## Phase 11 — Repository Coordination Contract + +**Status:** Complete + +- Implementation record: + [v1.13.11-phase11-repository-coordination-contract.md](v1.13.11-phase11-repository-coordination-contract.md). +- The exclusive-only contract, canonical container-namespace identity, stable + errors, diagnostic owner metadata, explicit lifecycle, and non-acquiring CLI + operation policy are implemented with fake-based tests. +- Native locking, CLI acquisition, subprocess contention, and live-GC barrier + proof remain unimplemented. BKC-016 remains `Deferred — documented`. +- Benchmark governance now separates hard four-profile candidate integrity + from four-profile hosted timing advice. Historical v1.9 threshold crossings + remain visible but cannot fail a valid timing job; invalid evidence still + fails. Hard performance enforcement is deferred to separately authorized + controlled infrastructure. +- Binary-identical paired run `30696834430` accepted the bounded functional + evidence architecture and rejected hosted paired timing authority. No paired + required job, production mode, manifest, or numeric paired policy is added. +- Exact-head validation and closure evidence are recorded in + [the Phase 11 closure evidence](v1.13.11-phase11-closure.md). BKC-016 remains + `Deferred — documented`. At Phase 11 closure, Phase 12 was still + unauthorized; its later implementation is recorded below. + +## Phase 12 — Cross-Platform Repository Lock Implementation + +**Status:** Complete + +- Closure record: + [v1.13.11-phase12-closure.md](v1.13.11-phase12-closure.md). +- Phases 12A–12F implement the safe control namespace, diagnostic owner + metadata, process-global non-reentrancy, Unix and Windows native locks, the + production Coordinator and CLI lifecycle, stable public errors, and the + direct-caller/Restore/Verify/GC contract. +- Phase 12G and reconciliation run `31258313120` prove native runtime and the + production Coordinator lifecycle on Linux, macOS, and Windows, with no + selected native test skips and a green Required Gate. +- Linux independent-process contention was proven by the bounded early Phase + 13A reconciliation. The killed-holder, live-GC, and dedicated PostgreSQL + advisory-session work that remained at Phase 12 closure is now recorded in + the completed Phase 13 below. BKC-016 remains `Deferred — documented`. + +## Phase 13 — Multi-Process Contention and Live-GC Barrier Proof + +**Status:** Complete + +- Closure record: + [v1.13.11-phase13-closure.md](v1.13.11-phase13-closure.md). +- Phase 13A reconciled the G6 + successful-overlap assumption with the exclusive fail-fast Lease while + preserving and strengthening data-integrity coverage, and proved + deterministic Linux independent-process contention. +- Phase 13B proved Linux killed-holder release and immediate reacquisition + without application cleanup, sleep, or retry. +- Phase 13C proved real Linux live-GC cross-process exclusion and pinned the + PostgreSQL advisory lock to one dedicated session with fail-closed cleanup. +- macOS/Windows subprocess semantics remain not separately proven and outside + the v1.13.11 acceptance contract. BKC-016 remains `Deferred — documented`. + +## Phase 14 — Container Range and Header Consistency Hardening + +**Status:** Complete + +- Evidence record: + [v1.13.11-phase14-container-range-header-consistency.md](v1.13.11-phase14-container-range-header-consistency.md). +- Outer container ranges are validated before allocation or filesystem I/O, + payload reads cannot overlap the fixed header, and physical appends use + overflow-safe capacity checks. +- Supported v0/v1 headers require a valid maximum; header, catalog, and physical + sizes must agree; packed reads use the persisted catalog maximum rather than + current process configuration. +- On-disk bytes, schemas, migrations, public APIs, CLI behavior, coordination, + decompression, and JSON rendering remain unchanged. + +## Phase 15 — Bounded Decompression + +**Status:** Complete + +- Evidence record: + [v1.13.11-phase15-bounded-decompression.md](v1.13.11-phase15-bounded-decompression.md). +- Identity and zstd codecs validate the exact expected size against the fixed + 4 MiB ceiling before decoder creation or destination allocation. +- Zstd whole-buffer output and decoder memory/window resources are bounded; + Restore, system Verify, and Store semantic reuse inherit the shared path. +- On-disk bytes, schemas, migrations, public APIs, CLI behavior, encryption, + hashes, Phase 14 container handling, and Phase 16 JSON rendering remain + unchanged. + +## Phase 16 — JSON Integer Fidelity + +**Status:** Complete + +- Evidence record: + [v1.13.11-phase16-json-integer-fidelity.md](v1.13.11-phase16-json-integer-fidelity.md). +- The stable v1.7 stats, inspect, and simulate-GC renderers preserve exact + integer tokens recursively through their generic envelope conversion. +- Integers remain JSON numbers; fields, nesting, error behavior, coordination, + schemas, storage formats, public APIs, dependencies, and workflows remain + unchanged. + +## Phase 17 — Fail-Closed SQL Mutation Audit + +**Status:** Complete + +- Evidence record: + [v1.13.11-phase17-fail-closed-sql-mutations.md](v1.13.11-phase17-fail-closed-sql-mutations.md). +- All 70 non-DDL production mutations are inventoried and classified; the 20 + required-row gaps now fail closed through the shared internal cardinality + sentinel and validator. +- Valid zero-row cleanup, recovery, CAS, upsert, bulk, and GC semantics remain + unchanged. SQLite/PostgreSQL affected-row parity, rollback, and + no-physical-delete-on-GC-mismatch behavior are proven. +- Schemas, migrations, dependencies, public APIs, retry policy, coordination, + workflows, and Phase 12–16 behavior remain unchanged. + +## Phase 18 — Required Backend and Coordination CI Gate + +**Status:** Complete + +- Evidence record: + [v1.13.11-phase18-required-backend-coordination-ci.md](v1.13.11-phase18-required-backend-coordination-ci.md). +- Required CI now fails closed on selected PostgreSQL mutation-cardinality, + storage/recovery, and Linux independent-process, killed-holder, and live-GC + execution events. SQLite quality and native cross-platform paths remain + required; gate topology and benchmark governance remain unchanged. +- Final implementation head `eaa5896` passed CI run `31872189672`, Required + Gate job `94984536507`, and CodeQL run `31872189661`. +- Closure head `9c1fa524` passed CI run `31873436272`, Required Gate job + `94987546529`, CodeQL run `31873436304`, and CodeQL Aggregate job + `94985540614`. + +## Phase 19 — Validation Matrix and Release Evidence Reconciliation + +**Status:** Complete + +- Evidence record: + [v1.13.11-phase19-validation-evidence-reconciliation.md](v1.13.11-phase19-validation-evidence-reconciliation.md). +- Reconciled the authoritative validation matrix, backend claim matrix, + reusable release checklist, release train, aggregate trackers, proof + boundaries, deferred items, evidence links, and Phase 18 closure chronology. +- Phase 19 changes documentation only. Product code/tests, CI workflows and + enforcement scripts, schemas, migrations, dependencies, public APIs, and + historical Phase 0–18 evidence remain unchanged. +- BKC-016 remains `Deferred — documented`; benchmark integrity remains hard + required, hosted timing remains advisory, and hard timing-regression + enforcement remains deferred to controlled infrastructure. + +## Phase 20 — Final Exact-Head Local Release Gate and PR Authorization + +**Status:** Complete + +- The canonical Phase 20 contract is + [the v1.13.11 release gate](v1.13.11-release-gate.md). +- The immutable candidate is the commit containing that gate record; the + record intentionally does not embed its own SHA. +- One pull request to `main` is operationally authorized only after that exact + candidate passes candidate-head CI, Required Gate, CodeQL, and the complete + clean local Profile A gate. +- Runtime results remain external evidence. Any later tracked edit creates a + new candidate and requires complete hosted and local revalidation. +- Merge, tag, publication, and release-branch deletion remain unauthorized. diff --git a/docs/release/v1.13/v1.13.11-phase0-post-release-closure-correction-and-baseline.md b/docs/release/v1.13/v1.13.11-phase0-post-release-closure-correction-and-baseline.md new file mode 100644 index 00000000..8545d767 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase0-post-release-closure-correction-and-baseline.md @@ -0,0 +1,79 @@ +# Coldkeep v1.13.11 Phase 0 — Post-Release Closure Correction and Baseline + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** `0 — v1.13.10 Post-Release Closure Correction and v1.13.11 Baseline` +**Status:** Complete +**Branch:** `release/v1.13.11` +**Starting baseline:** `423c57815580c39bee4f79ecd81570e9cfa9d273` + +## Decision + +v1.13.10 remains an immutable, valid released closure-integrity and +CI-runtime-hygiene baseline, but it is no longer the final v1.x release. Its +tagged decision used a narrower definition of completion. A post-release +roadmap-to-code audit found remaining must-before-v2 architectural and backend +commitments, so the final-v1.x conclusion is superseded without rewriting the +historical record. + +## v1.13.10 provenance + +Local evidence confirms annotated tag `v1.13.10` targets +`423c57815580c39bee4f79ecd81570e9cfa9d273`. That commit is the 2026-07-19 +GitHub merge commit for PR #105, titled `Coldkeep v1.13.10 — v1.x Closure +Integrity and CI Runtime Hygiene`. + +Public GitHub evidence confirms: + +- Stable release: `Coldkeep v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene`. +- Published: July 19, 2026 at 18:01. +- Tag target: `423c57815580c39bee4f79ecd81570e9cfa9d273`; merge provenance: PR #105. +- Tag-triggered CI: run #502, successful, 19 jobs, 18m26s. +- `release/v1.13.10` is absent from the public branch list. + +The local GitHub CLI could not retrieve this evidence because its available +token was invalid; public GitHub pages were used as the independent source. + +## Restored train + +The authoritative train is restored as: + +- v1.13.11 — Safety and Backend Compatibility Gate Closure. +- v1.13.12 — Engine and Catalog Completion. +- v1.13.13 — Final v1.x and v2 Handoff Gate. + +This restoration is required because compatibility proof, repository +coordination, catalog/engine completion, and final roadmap-to-runtime evidence +remain substantive must-before-v2 work. The updated +[`v1.13.x-release-train.md`](v1.13.x-release-train.md) is authoritative. + +## Authorized Phase 0 files + +Phase 0 created the five v1.13.11 canonical trackers and updated only the +authorized changelog, root/release READMEs, release train, and v1.13.10 +trackers. No runtime, test, version, schema, migration, workflow, script, +checklist, or validation-matrix file is authorized or changed. + +## Local validation evidence + +The branch was confirmed as `release/v1.13.11` with a clean starting tree. +`HEAD`, `main`, and `origin/main` were all +`423c57815580c39bee4f79ecd81570e9cfa9d273`; each is an ancestor of the other, +so this branch starts at current main. Required release-state and documentation +checks are run and reported with this Phase 0 change; the tracker does not +pre-claim their outcome. + +## Residual unknowns + +- The local GitHub CLI token remains invalid, but public GitHub evidence above + independently confirms the released state and operational closure evidence. +- The existing release-state validator may not represent an active v1.13.11 + documentation baseline while executable version remains 1.13.10; any such + result is visible validation evidence, not a reason to alter the validator + or version in Phase 0. + +## Workflow and authorization + +All phases stay on `release/v1.13.11`; there is one final PR only after the +complete release gate, and one phase-scoped commit at a time. No executable +version change occurs before Phase 1. Phase 0 authorizes only Phase 1 — +`v1.13.11 Release Identity Activation`. diff --git a/docs/release/v1.13/v1.13.11-phase1-release-identity-activation.md b/docs/release/v1.13/v1.13.11-phase1-release-identity-activation.md new file mode 100644 index 00000000..6c87577b --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase1-release-identity-activation.md @@ -0,0 +1,50 @@ +# Coldkeep v1.13.11 Phase 1 — Release Identity Activation + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** `1 — v1.13.11 Release Identity Activation` +**Status:** Complete +**Branch:** `release/v1.13.11` +**Starting commit:** `c4a21620f7ef4c7490a5045ffaae75e2a53392cb` + +## Result + +Phase 1 activated the executable and release-control identity from `1.13.10` +to `1.13.11`. The starting source version was `1.13.10`; the final source +version is `1.13.11`. + +## Changed files + +- `internal/version/version.go` +- `internal/version/version_test.go` +- `PRE_RELEASE_CHECKLIST.md` +- `CHANGELOG.md` +- `docs/release/v1.13/README.md` +- `docs/release/v1.13/v1.13.11-scope.md` +- `docs/release/v1.13/v1.13.11-phase-list.md` +- `docs/release/v1.13/v1.13.11-validation-checklist.md` +- `docs/release/v1.13/v1.13.11-release-gate.md` +- `docs/release/v1.13/v1.13.11-release-train-reconciliation.md` +- `docs/release/v1.13/v1.13.11-release-state-validator-contract.md` +- `docs/release/v1.13/v1.13.11-phase1-release-identity-activation.md` + +## Validation evidence + +Before activation, `python3 scripts/validate_release_state.py --state auto --json` +reported a Git-context lifecycle mismatch because the branch named v1.13.11 +while the executable version remained `1.13.10`. After activation, the same +validator reports the development state with zero violations. + +Local validation ran `gofmt`, `go test ./internal/version`, human and JSON CLI +version commands, the release-state fixture suite and validator, documentation +and CI-policy checks, semantic consistency search, and Git diff checks. The +version behavior was verified as `coldkeep version 1.13.11` and a successful +JSON version envelope reporting `1.13.11`. + +No product or runtime behavior changed apart from version reporting. No backend, +schema, CI, locking, or v2.x work was started. No remote CI evidence is claimed. + +## Next authorization and limitations + +Phase 2 — Backend Compatibility Claim Matrix is the only authorized next phase. +This phase does not establish backend compatibility, release-gate completion, +or release readiness. diff --git a/docs/release/v1.13/v1.13.11-phase10-backend-transaction-row-lock-semantics.md b/docs/release/v1.13/v1.13.11-phase10-backend-transaction-row-lock-semantics.md new file mode 100644 index 00000000..d8b0bfc6 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase10-backend-transaction-row-lock-semantics.md @@ -0,0 +1,343 @@ +# Coldkeep v1.13.11 Phase 10 — Backend Transaction and Row-Lock Semantics + +**Status:** Complete +**Branch:** `release/v1.13.11` +**Implementation base:** `e4c77607dd6e4132b32f93d0f1344a5a297dea16` +**Implementation commit:** `ad82c9596acc69380d312a8769bc19d44fd49998` + +Phase 10 implementation `ad82c959` sits directly on Phase 9 closure evidence +`e4c77607`. Its exact-head CI has completed successfully. + +## Contract placement + +The five top-level contracts are: + +| Contract | Package and file | +| --- | --- | +| `TestBackendTransactionCommitRollbackAcrossBackends` | `db_test`; `internal/db/transaction_backend_contract_test.go` | +| `TestBackendForUpdateLockReleaseAcrossBackends` | `db_test`; `internal/db/transaction_backend_contract_test.go` | +| `TestBackendNowaitAndSkipLockedAcrossBackends` | `db_test`; `internal/db/transaction_backend_contract_test.go` | +| `TestBackendBlockedLockCancellationAcrossBackends` | `db_test`; `internal/db/transaction_backend_contract_test.go` | +| `TestContainerRowLockIntegrationAcrossBackends` | `container`; `internal/container/row_lock_backend_contract_test.go` | + +Each contract uses `backendtest.ForEach`, which creates exact `sqlite` and +`postgres` subtests. None uses `t.Parallel()`. SQLite runs in its normal +temporary-file, single-connection fixture. PostgreSQL, when enabled, runs in a +per-test scratch database that the harness closes and drops. + +## TXN-001–003 — common transaction behavior + +`TestBackendTransactionCommitRollbackAcrossBackends` compares semantic +outcomes, not raw SQL or driver text. It proves: + +- runtime backend detection and the expected capability flags; +- exact omission of `FOR UPDATE`, `FOR UPDATE NOWAIT`, and + `FOR UPDATE SKIP LOCKED` by SQLite, and exact clause selection by PostgreSQL; +- an inserted row is visible inside its transaction and remains visible after + commit; +- an update is visible inside its transaction, while rollback restores the + prior value; +- a non-nil uniqueness failure followed by explicit rollback leaves no partial + committed row; +- `RowsAffected()` is one for an existing row and zero for a missing row; +- pre-cancelled `BeginTx` and `ExecContext` calls classify through + `errors.Is(err, context.Canceled)`; +- the baseline row remains unchanged and a final `SELECT 1` proves connection + reuse. + +This is bounded common behavior, not a claim of identical isolation levels. +Constraint error strings are deliberately not compared across drivers. + +## TXN-004 — PostgreSQL `FOR UPDATE` + +The fixture table is `phase10_lock_contract`, with fixed rows `(1, "first")` +and `(2, "second")`. The PostgreSQL branch reserves distinct physical locker, +contender, and observer `*sql.Conn` values. + +For both commit release and rollback release, transaction A selects row 1 with +`FOR UPDATE` and asserts its identity and unchanged value. The contender +connection's `pg_backend_pid()` is captured before transaction B starts the +same conflicting query in a goroutine. An observer connection polls +`pg_stat_activity` for that exact PID until it is `active` with +`wait_event_type = 'Lock'`. Only then does A commit or roll back. A buffered +result channel and wait group prove B completes, selects `(1, "first")`, and +can commit. The complete scenario is repeated for both release mechanisms; +final ordered row assertions prove both fixture values remain unchanged. + +There is no synchronization sleep. Observation uses a 10 ms ticker only to +poll server state, with a 2 second observer deadline. Operations and the +completion wait are bounded at 5 seconds. Reserved connections are checked on +close, all transactions have rollback cleanup, and the package-level final +reuse query succeeds. + +The SQLite branch only asserts that the production helper returns the base +query, executes that clause-free query, selects `(1, "first")`, rolls back, +and remains reusable. It makes no SQLite blocking row-lock claim. + +## TXN-005 — PostgreSQL `NOWAIT` + +Transaction A holds row 1 with ordinary `FOR UPDATE`. While A still holds the +lock, transaction B executes `FOR UPDATE NOWAIT` under a 2 second context and +returns before A releases. The test extracts `*pq.Error` with `errors.As` and +requires SQLSTATE `55P03`; it never compares driver text. + +A subsequent query proves A is still valid. B's failed transaction is +explicitly rolled back, A is rolled back to release the lock, and a new +transaction successfully acquires row 1 with NOWAIT. Row identities, final +unchanged fixture values, transaction cleanup, and connection reuse are +asserted. + +SQLite omits the NOWAIT suffix. Its shared branch executes the normal ordered +candidate query and selects row 1; no NOWAIT semantic claim is made. + +## TXN-006 — PostgreSQL `SKIP LOCKED` + +The candidates are rows 1 and 2 ordered by ascending ID with `LIMIT 1`. +Transaction A locks lower ID 1. Transaction B executes the ordered +`FOR UPDATE SKIP LOCKED` query and completes while A still holds its lock, +excluding row 1 and returning row 2. After B closes and A releases, a later +transaction runs the same query and returns row 1. `QueryRowContext(...).Scan` +consumes and closes its internal rows; every transaction and physical +connection is subsequently closed. + +SQLite omits the suffix and runs the ordered base query, returning row 1. This +proves deterministic base-query ordering, not skip-lock behavior. + +## TXN-007 — blocked-lock cancellation + +Transaction A locks `(1, "first")`. Transaction B starts an ordinary blocking +`FOR UPDATE` through a goroutine, buffered result channel, and wait group. The +observer polls `pg_stat_activity` for B's exact backend PID and proves the +server reports a lock wait before the test calls the query context's cancel +function. + +The result must be either `errors.Is(err, context.Canceled)` or a +`*pq.Error` with SQLSTATE `57014` when the driver does not retain the context +sentinel; `ctx.Err()` must itself be `context.Canceled`. Observation is bounded +at 2 seconds and completion at 5 seconds. B attempts rollback; when cancellation +makes `database/sql` discard B's physical connection, `driver.ErrBadConn` and +the later `sql.ErrConnDone` close are accepted as that terminal cleanup outcome. +A releases, and a new transaction successfully locks and reads `(1, "first")`. The goroutine joins, +the fixture is unchanged, and the final reuse query proves no transaction or +connection starvation. This is cancellation proof, not timeout proof. + +SQLite proves only the pre-cancelled, clause-free query boundary and subsequent +reuse. + +## TXN-008 — production container integration + +`internal/container/row_lock_backend_contract_test.go` exercises the actual +unexported production helpers rather than a copied query: + +- `lockContainerRowNowaitWithRetry` runs with exactly one attempt and a 1 ms + base-wait argument. A real PostgreSQL row lock makes its NOWAIT statement + return `55P03`; the production `*pq.Error` classifier maps only that code to + `ErrContainerLockContention`. Its savepoint rollback and release preserve the + surrounding transaction, proven by a subsequent `SELECT 1` in that same + transaction. After the conflicting transaction commits, the helper + successfully locks the same row. +- `selectOpenContainerExcluding` uses fixed eligible container IDs 1001 and + 1002. With 1001 locked on PostgreSQL, the production SKIP LOCKED allocator + returns 1002. After release it returns 1001, and passing exclusion ID 1001 + returns 1002. SQLite executes the production clause-free ordered query, + returns 1001, and returns 1002 when 1001 is excluded. + +The one-attempt NOWAIT setup is deterministic: the production retry helper's +tiny sleep is reachable only between attempts, so this integration executes no +retry sleep. This operation-level coverage is necessary because DB helper +tests alone cannot prove the production savepoint recovery, error mapping, +allocator predicates, exclusion behavior, or candidate ordering. + +The fixture seed is deterministic, connections are dedicated and checked on +close, transactions have rollback cleanup, the final `SELECT 1` succeeds, and +both seeded rows still exist. + +## Synchronization, deadlines, and cleanup + +Synchronization consists of unbuffered coordination through transaction +ordering, buffered one-result channels, wait groups, exact PostgreSQL backend +PID observation, a 10 ms observer ticker, context cancellation, and explicit +transaction release. No arbitrary sleep is used to assume lock acquisition or +blocked state. + +Hard bounds are: + +- 2 seconds for server lock-wait observation; +- 2 seconds for the NOWAIT query context; +- 5 seconds for DB test operation contexts, connection acquisition, backend + PID queries, final row/reuse queries, and blocked-query completion; +- 5 seconds for each container backend subtest's complete context. + +There is no separate sleep-based cleanup timeout. Cleanup is deterministic: +contexts are cancelled, result goroutines join, `QueryRow` results are +consumed, explicit transactions commit or roll back, deferred rollback accepts +only `sql.ErrTxDone`, reserved connections close, and the backend harness +removes the temporary SQLite file or closes and drops the PostgreSQL scratch +database. + +## Stable classifications and intentional boundaries + +The tests use only `errors.Is(context.Canceled)`, `ctx.Err()`, +`*pq.Error.Code == "55P03"`, optional PostgreSQL `57014`, +`ErrContainerLockContention`, numeric affected-row counts, and a non-nil +constraint failure. Post-cancellation cleanup also accepts the standard +`driver.ErrBadConn` and `sql.ErrConnDone` sentinels. They do not compare PostgreSQL or SQLite error strings, +cross-driver constraint text, or scheduler timing. No public error taxonomy +was added. + +SQLite proof is deliberately limited to backend detection, commit, rollback, +read-own-write, explicit rollback after constraint failure, affected-row +reporting, pre-cancelled context behavior, connection reuse, omission of all +three PostgreSQL lock suffixes, and production container helper behavior over +ordered base queries. It does not claim row-level locking, NOWAIT, SKIP LOCKED, +busy/locked classification, second-writer contention, live GC, or PostgreSQL +isolation equivalence. SQLite connection limits and pragmas are unchanged. + +## Production scope + +No production code changed. In particular, there are no non-test Go changes +under `internal/db`, `internal/container`, `internal/storage`, +`internal/snapshot`, `internal/maintenance`, or `internal/engine`. + +Phase 10 does not change transaction options, isolation levels, pool settings, +SQLite busy timeouts or pragmas, advisory locking, live GC, row-lock helper +implementations, schemas, migrations, codecs, container/storage formats, +public APIs, repository/process coordination, or Phase 11 files. + +## CI execution proof + +The existing required PostgreSQL plain-codec step is: + +```bash +go test -race -count=1 -json \ + ./internal/testutil/backendtest \ + ./internal/catalog \ + ./internal/db \ + ./internal/engine \ + ./internal/maintenance \ + ./internal/container +``` + +`./internal/container` is added exactly once; no prior package was removed and +no workflow, job, matrix/codec leg, or duplicate package run was added. The +parser requires exact package/test matches with `Action: "pass"` for: + +- `TestBackendTransactionCommitRollbackAcrossBackends/postgres` +- `TestBackendForUpdateLockReleaseAcrossBackends/postgres` +- `TestBackendNowaitAndSkipLockedAcrossBackends/postgres` +- `TestBackendBlockedLockCancellationAcrossBackends/postgres` +- `TestContainerRowLockIntegrationAcrossBackends/postgres` + +Skipped events do not satisfy a selector. Missing events, missing package +events, malformed JSON, or an absent/empty evidence file fail. `set -o +pipefail` preserves `go test` failures through `tee`. +`scripts/audit_ci_enforcement.sh --local-only` requires the single container +package occurrence and all five literals while retaining all earlier +selectors. + +## Local validation + +`COLDKEEP_TEST_DB`, `COLDKEEP_TEST_DB_MAINTENANCE`, `DB_HOST`, `DB_PORT`, +`DB_USER`, `DB_PASSWORD`, `DB_NAME`, `DB_SSLMODE`, and +`COLDKEEP_DB_AUTO_BOOTSTRAP` were all unset. Every PostgreSQL subtest therefore +skipped with: + +```text +set COLDKEEP_TEST_DB=1 (with DB_* connection settings) to run PostgreSQL backend tests +``` + +All five `/sqlite` subtests passed in the focused, five-repeat, and race +profiles. All five `/postgres` subtests skipped; those skips are not +PostgreSQL proof. + +Final local results after the assertion tightening: + +| Validation | Result | +| --- | --- | +| Focused once | PASS; DB `0.075s`, container `0.027s`, wall `1.427s` | +| Focused `-count=5` | PASS; DB `0.303s`, container `0.108s`, wall `1.513s` | +| Focused race | PASS; DB `1.120s`, container `1.028s`, wall `3.342s` | +| Six packages | PASS; DB `0.477s`, container `0.150s`, storage `6.030s`, snapshot `1.095s`, maintenance `0.195s`, Engine `2.561s`; wall `9.995s` | +| Six packages with race | PASS; DB `1.834s`, container `1.165s`, storage `12.264s`, snapshot `3.047s`, maintenance `1.447s`, Engine `4.758s`; wall `16.262s` | +| `go vet ./...` | PASS | +| CI-pinned `golangci-lint` v2.6.2 | PASS, zero issues | +| Release-state validator unit suite | PASS, 48 tests | +| Smart quotes, validation matrix, versioned row writers, local CI audit, `git diff --check` | PASS | + +The exact release validator JSON is: + +```json +{"status":"ok","validator":"coldkeep-release-state","state":"development","active_version":"1.13.11","violations":[],"error":null} +``` + +The default Go cache was read-only in the execution environment, so validation +used `GOCACHE=/tmp/coldkeep-phase10-gocache`. This affected only tool cache +placement. + +## Exact-head CI evidence and remaining boundaries + +Exact-head CI run `30148670910` executed at `ad82c959`. Plain correctness job +`89655223183` recorded passes for all five required PostgreSQL parent events; +quality, both correctness codecs, stress, long-run, adversarial, smoke, +compatibility, benchmark, and cross-platform evidence completed successfully. +The aggregate CI Required Gate `89656972706` passed. + +The first uncompressed benchmark attempt, job `89655633860`, failed its +existing workers=1 thresholds: `store-large-file` measured 7.22 Mbps against a +9.91 Mbps reference (27.1% regression), and `store-many-small-files` measured +0.04 Mbps against 0.05 Mbps (10.7% regression). No Phase 10 implementation, +benchmark baseline, threshold, fixture, or benchmark code changed. The same +exact-head benchmark was manually rerun as job `89656813012` and passed; its +dependent required gate then passed. This is **transient benchmark variance +resolved by a successful same-head rerun**, not a Phase 10 production +regression. + +Phases 0–10 are Complete. Phase 11 is Next but not started. + +- BKC-003: `Backend-specific — proven` +- BKC-014: `Backend-specific — proven` +- BKC-015: `Backend-specific — proven` +- BKC-016: `Deferred — documented` + +BKC-003 proves backend detection, PostgreSQL supported-clause emission and +execution, SQLite clause omission, and tested capability routing. BKC-015 +proves the documented common transaction guarantees; PostgreSQL `FOR UPDATE`, +`55P03` NOWAIT, SKIP LOCKED, cancellation, and production container helpers; +and SQLite's intentional clause-omission boundary. It excludes identical +isolation, SQLite row locks/NOWAIT/SKIP LOCKED, repository/process +coordination, advisory-lock session ownership, live-GC barriers, and +deadlock/serializable retry behavior. + +PostgreSQL advisory locks remain session-scoped while current acquisition and +release use pooled `*sql.DB` calls; that concern remains uncorrected and +Phase-13-owned. Ignored affected-row counts remain Phase-17 audit candidates. + +## Changed-file inventory + +Tests: + +- `internal/db/transaction_backend_contract_test.go` +- `internal/db/transaction_backend_contract_helpers_test.go` +- `internal/container/row_lock_backend_contract_test.go` + +CI: + +- `.github/workflows/ci.yml` +- `scripts/audit_ci_enforcement.sh` + +Evidence and trackers: + +- `docs/release/v1.13/v1.13.11-phase10-backend-transaction-row-lock-semantics.md` +- `CHANGELOG.md` +- `docs/release/v1.13/README.md` +- `docs/release/v1.13/v1.13.11-backend-compatibility-claim-matrix.md` +- `docs/release/v1.13/v1.13.11-phase-list.md` +- `docs/release/v1.13/v1.13.11-validation-checklist.md` +- `docs/release/v1.13/v1.13.11-scope.md` +- `docs/release/v1.13/v1.13.11-release-gate.md` + +There are no files outside this inventory. Recommended commit message: + +```text +test: prove backend transaction and row-lock semantics +``` diff --git a/docs/release/v1.13/v1.13.11-phase11-benchmark-gate-integrity-remediation.md b/docs/release/v1.13/v1.13.11-phase11-benchmark-gate-integrity-remediation.md new file mode 100644 index 00000000..ced52cd7 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase11-benchmark-gate-integrity-remediation.md @@ -0,0 +1,291 @@ +# Coldkeep v1.13.11 Phase 11 — Benchmark Gate Integrity Remediation + +**Status:** Outcome E preserved; hosted timing authority rejected; final +integrity-hard/timing-advisory policy implemented locally +**Phase:** 11 release-gate blocker +**Runtime scope:** Benchmark-only; no repository command behavior changes + +## Reason for the remediation + +> The absolute baseline/calibration and hosted paired-performance proposals in +> this document are historical remediation stages. The final policy is +> `v1.13.11-phase11-benchmark-governance-policy.md`: hard candidate-only v2 +> integrity plus operationally required, informational hosted timing. Raw +> schema v2, diagnostic-final-state schema v2, and Outcome E evidence policy v2 +> remain authoritative. No hosted timing input has production performance +> authority. + +Exact-head Phase 11 CI run `30151516328` measured +`store-many-small-files` at 721 ms and then 802 ms against one historical +668 ms observation. Both failures were valid under the old gate, but an +interleaved ten-sample parent/head investigation found the Phase 11 head 4.58% +faster by median, approximately 7–8% CV, direction-reversing blocks, and a +bootstrap interval spanning roughly -10.2% to +9.2%. No Phase 11 production +performance regression was proven. This establishes investigation Outcome D: +the original workload/environment lacks the stability needed for a truthful +single-sample release decision. + +The investigation instead confirmed that the old gate used one sample, +double-reported one elapsed-time change as duration and derived throughput, +accepted incomplete evidence, lacked environment provenance, and referenced +manifests with stale hashes. + +## Rejected paired qualification and bounded correction + +Binary-identical paired run `30687988675` preserved complete none-w1 and +zstd-w1 evidence and partial none-w4/zstd-w4 prefixes. All completed workers=1 +performance medians remained inside 0.95–1.05 with MAD at most 2.5%; all 342 +preserved workers=4 case rows retained valid counters and equal hard state. +This keeps same-job pairing viable and rejects hosted instability as the primary +cause. + +The v1 workload was not bounded: none-w1 took 37:53, none-w4 projected to about +54–55 minutes, and zstd-w4 projected to about 45–46 minutes. The outer +45-minute timeout killed both w4 samplers before reports/checksums, and the +decision command failed on missing checksums without owning a failure artifact. +The launcher also exposed encryption material, local database values, usernames, +DSN components, and runner paths in step environment displays. The selected +result is Outcome F: combined fixture and lifecycle correction required. + +New immutable `ci-paired-w1-v2` and `ci-paired-w4-v2` fixtures use a 64 MiB +large file and 400 one-KiB small files. They preserve 400 and 800 mixed files +respectively, the mixed range, all nine cases, fixed seeds, isolation, and ten +pairs. The old v1 identifiers remain supported for historical evidence. + +The sampler now owns a strict 35-minute monotonic deadline, caps the 600-second +command timeout by remaining budget, terminates/reaps the active process group, +performs compensating cleanup, and emits a checksummed prefix-only +`DIAGNOSTIC_TIME_BUDGET_EXCEEDED` artifact. The decision command owns its output +before validation and emits a checksummed failed decision without claiming +missing evidence was verified. A reusable audit enforces confidentiality and +the 35/45-minute lifecycle on any separately authorized launcher source; this +correction does not add a workflow. + +The first launcher-only v2 attempt, run `30693345495`, then exposed a narrower +launcher contract defect. The workflow created the exact path passed to every +`sample --output-dir`; the harness correctly rejected each existing path before +sampling. Consequently there is no v2 fixture, paired-statistic, hard-state, +counter, cleanup, or time-budget observation from that run. All four profile +artifacts were absent. The decision command correctly emitted its checksummed +failure-shaped decision and did not label missing profile evidence verified. + +Output ownership is now explicit: a workflow may create a contained parent, +but the sample or decision child must not exist at invocation. The launcher +asserts child nonexistence, the harness creates and exclusively owns it, and +artifact upload names that exact result. Static audit rejects direct or +indirect child creation, population, extraction, checkout, deletion/recreation, +profile sharing, traversal, symlink destinations, checkout/workspace roots, +and upload-path substitution. The decision harness now rejects existing empty +children just as the sample harness already did; no compatibility bypass was +added. + +Corrected binary-identical run `30696834430` then completed all four bounded v2 +profiles in 23:53–29:05. Its five artifacts verified full inventories, +checksums, equal hard state, valid counters, and `198/198` cleanup per profile. +This accepts the fixtures, schema-v2/Outcome-E functional evidence, deadlines, +cleanup, confidentiality, and artifact architecture. It rejects hosted paired +timing as a 5% hard endpoint: whole-command stability was profile-dependent and +did not protect against case-local outliers. + +Commit-B CI run `30698010935` independently rejected historical absolute timing +authority. About 66 minutes after accepted run `30695818412`, the identical +source tree, fixture, execution profile, Go version, architecture, and +PostgreSQL digest reported eight regressions in four cases. The final selection +is integrity hard and hosted timing advisory; controlled hard performance +enforcement is deferred. + +Confidentiality policy distinguishes fixed GitHub container aliases from +generated evidence. `/github/workspace`, `/github/runner_temp`, `/github/home`, +and `/github/workflow` are allowed platform runtime metadata. Actual host paths, +dynamically generated roots, credentials and keys, database/DSN components, +ports, container/database identifiers, benchmark namespaces, build paths, and +evidence paths containing generated identifiers remain confidential. They must +be created with tracing disabled, masked before possible output, passed directly +to subprocesses, and omitted from YAML environment displays, outputs, +provenance, stderr, and artifact names. + +No replacement launcher remains in the repository. Corrected bounded-v2 run +`30696834430` completed the separately authorized qualification lifecycle; +fixture sizes, order, pair count, deadline, and statistical bounds were not +adapted after observing results. + +Preserved calibration run `30176935742` subsequently established a narrower +integrity problem. Workers=1 was stable. Workers=4 varied selected container +open, close, and fsync counts in `restore-many-files`, `stats-inspect`, and, for +some zstd samples, `snapshot-creation` and `verify-system-deep`. The preserved +GitHub artifacts are intact raw diagnostics, but they predate the final-state +diagnostic object and cannot become corrected-contract calibration evidence. + +## Diagnostic bootstrap + +The bootstrap adds the fixed `ci-stable-v1` calibration candidate: + +- seed `1701`; +- one 96 MiB large file; +- 600 many-small files of 1 KiB; +- 400 deterministic mixed files from 1 KiB through 256 KiB; +- remove-every value `4`; +- the existing ordered nine-case suite; +- a separate temporary PostgreSQL database, repository, container root, and + data tree for every case. + +`ci-stable-v1` accepts only `repeat=1`. Independent sampling belongs to +`scripts/benchmark_gate.py`; ordinary `small`, `medium`, and `large` semantics +remain unchanged. Gate samples omit the command's historical pre-report +determinism suites because each external sample is already independent and +strictly validates fixture identity and counters. + +For each case, repository/data directories and the PostgreSQL database are +created before the case timer starts. The timer includes deterministic dataset +generation, ordinary command-driven schema bootstrap, every child-process +startup, and the Store/Restore/Snapshot/GC/Stats/Verify work for that case. It +excludes database creation and deletion, repository/data-tree creation and +cleanup, I/O-counter parsing, aggregate JSON rendering, and sampler overhead. +The separately recorded full-command duration includes benchmark-process +startup, all nine case timers, per-case database and filesystem setup/cleanup, +and raw JSON rendering; it excludes the sampler's post-exit validation and +aggregate rendering. + +Benchmark JSON now emits one top-level envelope and includes raw schema version +2 plus a deterministic fixture descriptor. The legacy CLI `--compare` path is +not the required release gate and now rejects empty, duplicate, reordered, or +different case sets. + +The separately versioned `diagnostic_final_state` schema 2 is observed after +each case and before cleanup. It contains sanitized canonical logical-file, +ordered chunk-graph, restored-tree, snapshot-membership, GC, verification, and +physical-content evidence plus a separate physical-layout digest. It excludes +database names and identifiers, DSNs, credentials, usernames, random or +temporary paths, and raw internal IDs. Observer failure still runs cleanup. + +## Outcome E conclusion + +The bounded local diagnostic under `/tmp/coldkeep-phase11-local-v2` compared +one workers=1 reference and ten workers=4 measurements for both `none` and +`zstd`. All 22 raw samples were retained. There were zero hard mismatches: +fixture identity, processed totals, logical-file state, ordered chunk graphs, +restored trees, snapshot membership, GC semantics, verification totals, +canonical physical content, operation outcomes, and cleanup remained exact. + +The workers=4 variation was limited to balanced open/close and matching fsync +changes of one event in the cases identified above. Container allocation counts +varied with the same cases, and the placement/layout digest remained +informational after canonical physical content matched. This selects: + +```text +Outcome E — calibration evidence contract is over-constrained. +``` + +No production correctness defect, fixture defect, isolation defect, or +aggregation defect was proven. + +## Corrected evidence policy + +Evidence policy version 2 is an explicit allowlist with five nonempty classes: + +- `hard_equal`: schema/report status, capture source/binary and hard environment + identity, complete profile and fixture identity, ordered cases and seeds, + warmup/sample counts, per-case processed file/byte totals, operation results, + expected restored totals, all semantic diagnostic fingerprints/totals, + canonical physical payload content, cleanup success, and zero leaked + databases/processes/temporary resources; +- `derived_equal`: recomputed throughput and statistics, aggregate row totals, + sample ordering/count relationships, command p95, duplicated outer/I/O + counters, snapshot-write totals, open/close balance, and applicable hashes; +- `bounded_nonnegative`: mandatory signed-64-bit container append/open/close, + fsync, byte-read, and byte-written observations with type, overflow, + contradiction, and balance checks but no invented percentage bound; +- `informational`: retained timings and counter distributions, allocation + counts, container bytes, physical-layout digest, host load/free disk, and + runner-image warnings; +- `excluded_sensitive`: credentials, keys, DSNs, usernames, database names, + repository/temporary paths, sensitive arguments, environment dumps, and raw + internal IDs. + +Raw schema 2 omits a zero-valued snapshot metadata write counter; the corrected +policy preserves that established behavior, normalizes omitted zeroes, retains +nonzero samples, validates operation compatibility, and recomputes the +top-level sum. Missing `diagnostic_final_state`, wrong diagnostic schema, +malformed or incomplete fields, unknown correctness or informational +extensions, non-finite values, and sensitive names or values fail closed. + +Canonical physical content and allocation layout remain separate. The hard +placement-independent digest covers chunk/payload identity and size plus +codec/compression transforms, while container IDs, block placement, allocation +counts, container bytes, and the layout digest are informational. A content, +size, transform, reachability, missing, duplicate, or corruption difference is +never made informational. + +## Superseded absolute calibration decision + +The following fixed-calibration design is retained only to explain the rejected +path. It is not an authorized launcher or acceptance contract. + +The manual-only benchmark workflow runs two independent replicas of every +`none/zstd × w1/w4` profile. Each replica uses one excluded warmup and exactly +ten measured samples. Calibration passes only if: + +- all hard-equal fixture, profile, semantic final-state, operation, and cleanup + evidence remains identical; +- all scheduling-sensitive counters remain mandatory, internally valid, and + fully retained without requiring exact cross-sample equality; +- every case median is at least 5,000 ms; +- every full-command p95 is no more than 120 seconds; +- each ten-sample distribution and its fixed odd/even five-sample partitions + satisfy `MAD/median <= min(2%, threshold/2)`; +- independent replica medians differ by no more than 5%. + +Samples are never discarded. Failure stops the effort; it does not authorize +adaptive sampling, fixture changes, baseline refresh, or a threshold change. + +## Replaced future gate contract + +The intended gate is now the same-run paired architecture documented in +`v1.13.11-phase11-paired-benchmark-gate-contract.md`. It uses +candidate/reference ratios within five fixed pairs, not independent absolute +medians. Existing v1.9 thresholds are not reused. + +The superseded single-binary aggregate schema v2 records raw durations, median, +mean, min/max, sample standard deviation, MAD, MAD ratio, CV, hard final-state +evidence, every operational sample and distribution, source and binary hashes, +resolved runner information, exact Go/PostgreSQL versions, and the database +image digest. +Credentials, DSNs, usernames, arguments, and temporary paths are excluded. The +paired gate adds a separate comparison schema v1 and does not call it aggregate +schema v3. + +## Governance and current boundary + +`.github/workflows/benchmark-baseline.yml` remains manual-only and read-only. It +captures artifacts but never commits or refreshes a baseline. Required CI now +uses separate `benchmark-integrity` and `benchmark-timing-advisory` matrices. +The former is hard functional authority; the latter retains historical +threshold crossings as visible advice and verifies its report/exit mapping. + +The historical `small` observation uses the ordinary benchmark path and may +legitimately omit the Go `omitempty` diagnostic final state and zero-valued +operational counters. Its timing validator normalizes only those defined +zero-value omissions, validates optional final-state structure when present, +and otherwise fails closed on unknown fields. Hard final state, repeat +equality, semantic content, and explicit per-case/repeat cleanup remain solely +owned by `benchmark-integrity`; successful historical command completion only +establishes that command's temporary database cleanup. + +Exact-head run `30700383632` failed before timing evaluation because the hosted +comparator incorrectly imposed the integrity-only final-state requirement on +all four ordinary `small` reports. This was a validator/serialization contract +mismatch, not a product regression or timing warning. Evaluator failures now +emit a hard machine-readable `BENCHMARK_TIMING_EVALUATION_FAILURE`, are +summarized and checksummed, and fail after artifact finalization. Valid timing +warnings remain advisory and successful. + +The v1.9 measurements and 3%/5% thresholds are unchanged. Their manifests are +known to require governed repair, but no hash or baseline content is changed +by this correction. Historical fixture values are unchanged; the v2 fixtures +retain their immutable identifiers. No governed hard timing baseline exists +under the corrected contract. Paired production remains hard-disabled and no +reference manifest or numeric paired threshold policy is created. + +Phase 11 is **Complete** with exact-head evidence recorded in +[the Phase 11 closure evidence](v1.13.11-phase11-closure.md). BKC-016 remains +`Deferred — documented`, and Phase 12 remains unauthorized. diff --git a/docs/release/v1.13/v1.13.11-phase11-benchmark-governance-policy.md b/docs/release/v1.13/v1.13.11-phase11-benchmark-governance-policy.md new file mode 100644 index 00000000..d8d2422a --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase11-benchmark-governance-policy.md @@ -0,0 +1,184 @@ +# Coldkeep v1.13.11 Phase 11 — Final Benchmark Governance Policy + +**Status:** Complete; exact-head CI and CodeQL acceptance recorded +**Phase:** 11 is Complete +**Policy:** Hard integrity, hosted timing advisory +**Performance enforcement:** `deferred_to_controlled_infrastructure` + +## Decision + +Required CI separates benchmark correctness from hosted-runner timing: + +- `benchmark-integrity` is hard-required. Its four jobs run `none-w1`, + `none-w4`, `zstd-w1`, and `zstd-w4` against the worker-bound bounded v2 + fixtures. Each job captures exactly two candidate-only samples, with no + warmup and a 600-second command ceiling. Repeat hard state, semantic content, + counters, cleanup, confidentiality, inventory, and checksums must validate. +- `benchmark-timing-advisory` is operationally required but has informational + timing authority. Its four jobs retain the historical `small` command and + v1.9 comparison, validate the schema-v2 candidate and frozen legacy baseline, + preserve all threshold crossings, verify the comparator exit/report mapping, + and succeed for a valid timing warning. The ordinary historical path does + not emit `diagnostic_final_state`; hard semantic final-state authority remains + exclusively with `benchmark-integrity`. +- `CI Required Gate` requires both job families to finish successfully. + Integrity failure, malformed evidence, comparator failure, missing files, + checksum failure, skip, or cancellation remains blocking. A verified timing + warning is not an integrity failure. + +The policy is Option A: integrity hard, timing informational. Hosted timing is +never labelled `PASS`. Historical thresholds remain visible as advice and are +not production performance authority. + +## Evidence supporting the decision + +Binary-identical paired qualification run `30696834430` completed all four v2 +profiles in 23:53–29:05, below the 35-minute internal deadline. All five +artifacts verified; every profile contained two warmups, ten balanced pairs, +22 reconstructed invocations, schema-v2 raw and final-state evidence, equal +hard state, valid counters, `198/198` cleanup, and valid checksums. Both binaries +had SHA-256 `46a22757…9a9e0e`. + +The run accepted the bounded fixtures and evidence lifecycle, but rejected +GitHub-hosted whole-command A/A timing as a 5% hard performance endpoint. +Only `none-w1` and `zstd-w4` had stable whole-command ratios; `none-w4` aged +about 27%, `zstd-w1` retained strong temporal/position variation, and a +case-specific `zstd-w4` `stats-inspect` outlier remained possible despite a +stable aggregate. + +Historical required-CI runs provide independent confirmation. Accepted run +`30695818412` and failed run `30698010935` used the same source tree, fixture, +execution profile, Go `1.25.12`, Linux/amd64, and PostgreSQL digest +`33f923…e5f20`; about 66 minutes apart, the latter reported eight regressions +in four cases. No Go source, fixture, build configuration, or executable +behavior changed. Hosted-runner variation is therefore not a truthful hard +release signal at the historical thresholds. + +## Confidentiality boundary + +Credentials, encryption keys, DSNs, generated database/role/container +identifiers, ports, benchmark namespaces, Coldkeep-controlled roots, and +evidence paths containing protected identifiers are prohibited. Fixed GitHub +aliases, public fixture/profile names, versions, source SHAs, public image +digests, action cache paths, and opaque generic `/tmp/go-build…` toolchain paths +are allowed. This is not a blanket `/tmp` allowance. Unknown or ambiguous +values fail closed or require explicit review. The paired launcher audit and +evidence validators retain this policy even though no paired launcher is +tracked. + +## Hard integrity contract + +`scripts/benchmark_gate.py integrity` owns a previously nonexistent output +child and fixes: + +- two measured candidate invocations and zero warmups; +- `ci-paired-w1-v2` for workers 1 and `ci-paired-w4-v2` for workers 4; +- Go `1.25.12`, PostgreSQL digest `33f923…e5f20`, codec/compression, pipeline + depth 1, deterministic mode, case isolation, order, and seeds; +- 600 seconds per invocation. + +`BENCHMARK_INTEGRITY_PASS` requires two valid raw reports, equal repeat hard +state, valid semantic and operational counters, complete cleanup, a strict +aggregate, a strict `benchmark-integrity.json`, sanitized stderr captures, and +exhaustive checksums. The authority record is +`integrity_authority: true, performance_authority: false`. + +Controlled failures return nonzero and emit +`BENCHMARK_INTEGRITY_FAILURE`. Failure evidence records the completed prefix, +truthful active/incomplete invocation, reason, and cleanup state and makes no +aggregate claim. Command, timeout, schema, fixture, execution, order, seed, +namespace, catalog, chunk, restore, snapshot, GC, verification, canonical +content, counter, cleanup, confidentiality, inventory, and checksum defects are +hard failures. Scheduling-sensitive allocation and layout remain retained +informational observations under Outcome E. + +## Timing advisory contract + +`scripts/validate_regression_thresholds.py check --policy hosted-advisory` +strictly validates the candidate schema-v2 `small` envelope, execution profile, +nine-case order, counters, totals, and derived throughput. Successful completion +of the historical benchmark command establishes its temporary database cleanup; +explicit repeat/case cleanup and semantic-state evidence remain owned by the +hard integrity matrix. A present optional `diagnostic_final_state` is validated, +but absence is valid for this ordinary `small` observation. The comparator +validates the frozen baseline's established legacy shape without describing it +as schema-v2 evidence. It records raw input hashes, violations, deltas, +thresholds, classification, and summary. + +Exit codes are: + +| Code | Classification | +| ---: | --- | +| 0 | `BENCHMARK_TIMING_WITHIN_REFERENCE` | +| 10 | `BENCHMARK_TIMING_WARNING` | +| 11 | `BENCHMARK_TIMING_UNSTABLE` | +| 12 | `BENCHMARK_TIMING_NOT_EVALUATED` | +| 2 | `BENCHMARK_TIMING_EVALUATION_FAILURE` | + +`BENCHMARK_TIMING_UNSTABLE` requires a statistically capable input and is not +inferred from one historical observation. `BENCHMARK_TIMING_NOT_EVALUATED` is +reserved for an intentionally unavailable, allowlisted non-integrity reason; +missing or malformed evidence and comparator errors cannot use it. +`BENCHMARK_TIMING_EVALUATION_FAILURE` is a hard job failure with +`status: failed`, informational authority, a sanitized error category/message, safely +established input hashes when readable, and no timing or production claim. + +Required CI disables shell `errexit` only around the comparator, captures its +exact code, immediately restores `errexit`, and runs +`verify-advisory-exit`. Broad suppression and arbitrary-code allowlists are +forbidden. Summaries call threshold crossings warnings, never passes. +Codes 0, 10, 11, and 12 are narrowly accepted after report/code verification. +Code 2 is verified, summarized, checksummed, and uploaded before the job fails. + +Archived exact-head run `30700383632` failed all four timing-advisory jobs on +`row store-large-file fields mismatch`: the comparator required diagnostic +final state that the ordinary `small` serializer legitimately omitted. The run +therefore shows an implementation contract mismatch, not a product regression +or timing-warning failure. The corrected exact head was accepted; its evidence +is recorded in [the Phase 11 closure evidence](v1.13.11-phase11-closure.md). + +## Artifacts and audit + +Every integrity and timing job uploads its exact owned directory with +`if: always()` and `if-no-files-found: error`. Integrity artifacts include two +raw reports and stderr captures, aggregate evidence, the integrity decision, +and checksums. Timing artifacts include the historical raw observation, +advisory report, and checksums. Required jobs verify checksums before upload. + +The CI audit enforces four profiles in each family, fixture/worker binding, +two-sample/no-warmup integrity policy, the 600-second ceiling, non-performance +authority, exact advisory exit verification, absence of broad suppression, +separate Required Gate dependencies, always-upload behavior, and missing-file +rejection. It also preserves the existing prohibitions on required paired jobs, +paired production mode, a paired reference manifest, and a paired threshold +policy. + +## Frozen and retained components + +All v1.9 baseline JSON, manifests, thresholds, and reports remain byte-for-byte +unchanged. The manual read-only baseline workflow remains optional historical +diagnostic tooling. Required CI may cite those files only as +`historical_v1.9_absolute` advisory inputs. + +The v2 fixtures, paired sampler/decision, strict artifacts, lifecycle controls, +confidentiality checks, and launcher audit remain reusable diagnostic or +controlled-runner tooling. Run `30696834430` accepts that functional evidence +architecture while rejecting hosted paired timing authority. Required CI has no +paired job. Production paired sampling remains hard-disabled; no manifest or +numeric paired threshold policy exists. + +## Controlled-infrastructure deferral + +Hard performance enforcement requires a separately authorized design covering +immutable hardware/storage and OS/kernel/toolchain/database identity, exclusive +scheduling, load/thermal/health envelopes, secure fork and credential handling, +reference capture and rollover, drift monitoring, maintenance, and +availability. Activation also requires a binary-identical qualification with +acceptable false-positive behavior and separately reviewed reference and +numeric threshold policies. None is authorized here. + +Phase 11 is **Complete** with accepted exact-head local validation, four +integrity passes, successful advisory evaluation, complete artifacts, and green +CI and CodeQL recorded in [the closure evidence](v1.13.11-phase11-closure.md). +BKC-016 remains `Deferred — documented`. Phase 12 remains unauthorized and not +started. diff --git a/docs/release/v1.13/v1.13.11-phase11-closure.md b/docs/release/v1.13/v1.13.11-phase11-closure.md new file mode 100644 index 00000000..02229317 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase11-closure.md @@ -0,0 +1,86 @@ +# Coldkeep v1.13.11 Phase 11 — Closure Evidence + +**Phase:** 11 — Repository Coordination Contract +**Status:** Complete +**Closure authority:** documentation and evidence only +**Accepted exact head:** `b08da99a8efe39b5c9cdb0bea59362304bd2fc09` + +## Closure decision + +Phase 11 is complete as the repository-coordination **contract and evidence +boundary** for v1.13.11. BKC-016 remains **Deferred — documented**: the +contract is recorded, but its native implementation and process-level proof are +not release claims. + +Phase 12 is not started or authorized. Phase 13 is not started. The v1.13.11 +release remains active and incomplete. + +## Implemented Phase 11 boundary + +The completed contract defines exclusive-only repository coordination, +canonical repository identity, control-root ownership, typed acquisition and +contention errors, owner metadata, lifecycle and non-reentrancy rules, lock +ordering, operation policy, and the coordination boundaries for restore, +verification, and garbage collection. Local fake-based contract tests cover +those semantics. + +This closure does not claim native OS lock acquisition, lock-artifact +acquisition, CLI runtime integration, process-contention proof, crash-release +proof, PostgreSQL advisory-session correction, live-GC proof (including SQLite +live-GC proof), or cross-host/network-filesystem behavior. Those boundaries +remain Phase 12 or Phase 13 work as documented in the coordination contract. + +## Benchmark-governance disposition + +The v1.13.11 policy keeps benchmark artifact integrity as a hard gate and +timing regression as an advisory comparison against frozen v1.9 references. +Controlled-infrastructure performance proof remains deferred. Paired benchmark +assets remain diagnostic evidence only: they do not activate a required CI +gate, modify a manifest, or alter any threshold authority. + +The earlier `30696807818` paired-benchmark architecture was accepted only as +functional diagnostic evidence; its 5% timing rule was rejected as release +authority. No rejected paired policy was activated by this closure. + +## Exact-head validation evidence + +At `b08da99a8efe39b5c9cdb0bea59362304bd2fc09`, local validation recorded: + +- 16 advisory-comparator test methods passed. +- 92 combined regression-threshold, benchmark-gate, and paired-benchmark-gate + tests passed. +- 48 release-state validator tests passed. +- `go test -count=1 ./scripts`, repeated script tests, `go test -count=1 ./...`, + `go test -race -count=1 ./...`, `go vet ./...`, shell checks, validation + matrix checks, row-writer audit, and CI-enforcement audit passed. +- The release-state validator reported development state with active version + `1.13.11` and no violations. + +Hosted acceptance at the same exact head recorded: + +| Evidence | Result | +| --- | --- | +| CI run `31242784813` | Success | +| Required Gate job `93068202252` | Success | +| CodeQL run `31242784806` | Success (Go, Python, Actions, and aggregate) | + +The four hosted benchmark profiles (`none-w1`, `none-w4`, `zstd-w1`, and +`zstd-w4`) each published `benchmark.json`, `timing-advisory.json`, and +`checksums.sha256`. Their checksums verified; every profile reported +`BENCHMARK_INTEGRITY_PASS` and `BENCHMARK_TIMING_WITHIN_REFERENCE`, with zero +timing-policy violations. + +## Handoff boundaries + +Phase 12 retains native Linux/macOS/Windows acquisition and CLI integration. +Phase 13 retains independent-process contention, crash-release, restore/live-GC +barrier, and PostgreSQL advisory-session ownership proof. Neither phase is +implemented, authorized, or advanced by this closure. + +## Final state + +- **Phase 11 — Repository Coordination Contract:** COMPLETE +- **BKC-016:** Deferred — documented +- **Phase 12:** Not started / unauthorized +- **Phase 13:** Not started +- **v1.13.11 release:** Active / incomplete diff --git a/docs/release/v1.13/v1.13.11-phase11-paired-benchmark-gate-contract.md b/docs/release/v1.13/v1.13.11-phase11-paired-benchmark-gate-contract.md new file mode 100644 index 00000000..29d4da31 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase11-paired-benchmark-gate-contract.md @@ -0,0 +1,339 @@ +# Coldkeep v1.13.11 Phase 11 — Paired Benchmark Gate Contract + +**Status:** Functional evidence architecture accepted; GitHub-hosted paired +timing authority rejected by bounded v2 qualification +**Phase:** 11 release-gate blocker +**Authority:** Diagnostic/controlled-runner tooling only; no hosted performance +authority, reference, threshold, or required paired job + +## Decision and boundary + +Coldkeep evaluated replacing absolute checked-in timing comparisons with a +same-job, same-run reference-versus-candidate comparison. Each of four independent +profiles builds an immutable reference and the exact candidate, runs both +against the same pinned PostgreSQL service in a fixed interleaved order, and +evaluates paired candidate/reference duration ratios: + +```text +none-w1 +none-w4 +zstd-w1 +zstd-w4 +``` + +No paired ratio currently carries performance authority. The bounded +qualification showed that hosted-runner whole-command ratios were not +consistently stable at 5%, so absolute runtime, host load, filesystem speed, +and hosted pair ratios are informational. Raw benchmark schema version 2, +`diagnostic_final_state` schema version 2, and Outcome E evidence policy version +2 remain unchanged. The paired result is a separate +`benchmark_paired_comparison` schema version 1; it is not aggregate schema v3. + +The separately reviewed harness implementation is limited to the two fixtures, +full-suite CLI routing, `scripts/paired_benchmark_gate.py`, and focused tests. +Diagnostic sampling and explicit-mode diagnostic aggregation are implemented; +a successful diagnostic result is `DIAGNOSTIC_QUALIFIED`, never production +`PASS`. This contract does not add or authorize a temporary workflow, reference +manifest, numeric threshold policy, baseline, remote run, required-CI change, +release-status change, or Phase 12 work. Production has no default reference or +threshold. The harness must keep production sampling and production decisions +explicitly hard-disabled, so merely adding files cannot enable authority. A +later separately authorized controlled-infrastructure design must add reviewed +inputs, trusted-base workflow integration, and deliberately enable that code +path. + +Binary-identical run `30696834430` completed all four v2 profiles before the +35-minute deadline with full inventories, checksums, equal hard state, valid +counters, and complete cleanup. It accepted this functional evidence and +lifecycle architecture. It rejected hosted paired timing authority because +whole-command stability varied materially by profile and aggregation could +hide case-specific instability. The final required-CI policy is documented in +`v1.13.11-phase11-benchmark-governance-policy.md`. + +## Fixture and case contract + +Both current fixtures use seed `1701`, 1 KiB many-small files, mixed file sizes from 1 +KiB through 256 KiB, `remove_every=4`, and case database isolation. + +| Field | `ci-paired-w1-v2` | `ci-paired-w4-v2` | +| --- | ---: | ---: | +| Large file | 64 MiB | 64 MiB | +| Many-small files | 400 | 400 | +| Mixed files | 400 | 800 | +| Required workers | 1 | 4 | + +Every invocation emits all nine cases in this order: + +```text +store-large-file +store-many-small-files +store-mixed-dataset +restore-large-file +restore-many-files +snapshot-creation +gc-after-churn +stats-inspect +verify-system-deep +``` + +The performance endpoints are `store-large-file`, +`store-many-small-files`, `restore-many-files`, `snapshot-creation`, +`gc-after-churn`, `stats-inspect`, and `verify-system-deep`. +`store-mixed-dataset` and `restore-large-file` remain mandatory correctness, +counter, and cleanup evidence and are never omitted. + +`ci-stable-v1` remains historical diagnostic compatibility and has no paired +performance authority. `ci-paired-w1-v1` (96 MiB/600/400) and +`ci-paired-w4-v1` (128 MiB/1,200/800) remain immutable and parseable, but the +rejected qualification proved them too expensive for the bounded diagnostic. +The v2 fixtures are accepted bounded integrity/diagnostic inputs, not +production performance workloads. Fixture values cannot adapt at runtime. Any +revision requires a new identifier and review before production use. + +## Execution inventory + +Excluded warmups are always `C R`. Production uses exactly five measured +pairs: + +```text +R C | C R | C R | R C | R C +``` + +The one bounded diagnostic uses exactly ten measured pairs by following the +five-pair block with its side-inverted block: + +```text +R C | C R | C R | R C | R C | +C R | R C | R C | C R | C R +``` + +Five pairs necessarily have a 3/2 first-position split; the inverted second +block makes the diagnostic split exactly 5/5. Pair membership and position are +assigned before execution. A missing, duplicate, reordered, discarded, or +additional invocation invalidates the profile. There is no adaptive +repetition, sample extension, or outlier removal. + +Both binaries share the runner, CPU, resolved image, architecture, pinned Go +toolchain, PostgreSQL service and digest, codec, compression, worker count, +pipeline depth, deterministic mode, and environment policy. Every case gets a +fresh database, repository, data tree, and container root. + +## Statistical contract + +For case `k` and pair `i`: + +```text +paired_ratio_i = candidate_duration_i / reference_duration_i +median_ratio = median(paired_ratio_1..paired_ratio_n) +regression_pct = (median_ratio - 1) * 100 +paired_mad_ratio_pct = + median(abs(paired_ratio_i - median_ratio)) / median_ratio * 100 +``` + +The implementation derives median and comparison boundaries from decimal +representations of raw durations. It never divides independently aggregated +medians. A production regression exists only when the unrounded +`regression_pct` is strictly greater than the governed case threshold; exact +equality passes. Improvements and zero deltas pass. Recomputed candidate +throughput is informational. + +Production stability is: + +```text +paired_mad_ratio_pct <= min(3%, regression_threshold_pct / 2) +``` + +Equality passes. Instability takes precedence over performance and becomes +`BENCHMARK_ENVIRONMENT_UNSTABLE`. + +No numeric production threshold is authorized or checked in. A future policy +must enumerate exactly the seven performance cases, be referenced by an +approved manifest and SHA-256, and set no case above 10%. The v1.9 absolute +thresholds cannot be reinterpreted as paired-ratio thresholds. + +## Bounded diagnostic qualification + +The separately authorized binary-identical diagnostic must run ten complete +pairs in all four profiles. It qualifies hosted gating only if: + +- every median ratio is within `0.95`–`1.05`, inclusive; +- every paired MAD ratio is at most `2.5%`; +- every command and pair is present and no timeout occurs; +- reference and candidate hard state match exactly; +- all operational counters are valid and cleanup is complete; +- each profile, including validation and complete report construction, finishes + strictly before the 35-minute internal deadline. + +Failure rejects 5% hosted-runner sensitivity. It does not authorize wider +thresholds, discards, automatic fixture changes, or baseline creation; the next +decision is benchmark policy or self-hosted infrastructure. + +The four-profile aggregation caller must explicitly select diagnostic scope: + +```text +paired_benchmark_gate.py decision --mode diagnostic \ + --profile none-w1= --profile none-w4= \ + --profile zstd-w1= --profile zstd-w4= \ + --output-dir +``` + +Both commands require an existing workflow-owned parent and a nonexistent +harness-owned output child. A launcher must create only the parent, assert +`test ! -e` for the exact child, pass that child through `--output-dir`, and +upload that same child after execution. It must not create, touch, populate, +extract into, check out into, delete/recreate, share across profiles, or replace +the child. The child cannot be `.`, a workspace root, a repository checkout, +a traversal, or a symlink destination. `sample` and `decision` each create the +accepted child and acquire exclusive artifact ownership before writing any +success- or failure-shaped evidence. + +Diagnostic decisions accept only diagnostic artifacts, require byte-identical +reference/candidate hashes within and across all profiles, reconstruct results +from checksummed raw evidence, and record `diagnostic_qualification`, +`diagnostic_only`, and `production_authority: false`. Median ratios outside the +inclusive 0.95–1.05 band classify `DIAGNOSTIC_REJECTED`; paired MAD above 2.5% +classifies `BENCHMARK_ENVIRONMENT_UNSTABLE`. At the exact 35-minute boundary the +sampler terminates and reaps the active process group and classifies +`DIAGNOSTIC_TIME_BUDGET_EXCEEDED`; this is diagnostic-only and never a product +regression. Statistical exact bounds still pass. + +## Correctness and Outcome E policy + +Every raw report validates schema version 2, diagnostic-final-state schema +version 2, fixture/execution identity, ordered cases, derived throughput, +signed nonnegative counters, counter relationships, open/close balance, +sensitive-field exclusion, and complete top-level recomputation. + +Within each binary, active logical namespace, logical catalog/statuses, chunk +graph, restored tree, snapshots, GC, verification, processed logical totals, +and canonical physical content remain exact. Those fields must also match +between reference and candidate. Canonical-content changes are correctness +regressions. + +Scheduling-sensitive operational counters remain mandatory, validated, and +fully distributed by binary but may differ. Container/block allocation, +container bytes, and physical-layout digest may differ only as already +classified informational after canonical physical content matches. Intentional +semantic changes require a new contract version and governed compatibility +evidence; there is no runtime waiver. + +## Reference governance + +Production authority will come from reviewed +`benchmarks/paired/reference-v1.13.json`, never a tag lookup, workflow input, +environment override, or artifact. Its strict schema records the release +train, exact reference SHA, trusted approval record, contract/evidence schema +versions, fixture/case inventories, execution order, pair count, and threshold +policy identity and digest. + +The reference must exist and be an ancestor of the exact event candidate SHA. +Pull requests and merge queues measure the tested merge SHA; pushes measure the +pushed SHA. Candidate changes to effective reference or threshold paths fail +ordinary jobs. Backports require explicit governance review. + +Bootstrap remains separately governed: + +1. Treat the first paired implementation commit as provisional. +2. Measure a launcher-only child against that parent with binary-identical + product sources. +3. Review all diagnostic artifacts. +4. In a later authorized manifest-only commit, pin the approved implementation + SHA and threshold policy. + +Reference rollover is a dedicated allowlisted manifest/documentation-only +change after the proposed reference passed the previous gate and became +reachable from trusted release approval. There is no automatic latest-tag +resolution. + +## Artifacts, decision, and failures + +Each complete profile artifact contains warmup and measured raw JSON/stderr, a complete +inventory, binary hashes, sanitized provenance, fixture/order identity, paired +statistics, counter distributions, hard-state result, cleanup totals, +classification, and `checksums.sha256`. Binary paths, database names, DSNs, +credentials, repository roots, environment dumps, and sensitive arguments are +excluded. A failed profile instead contains only its validated fixed-order +prefix, explicit active/incomplete invocation, elapsed/cancellation reason, +prefix hard-state/counter results, compensating-cleanup status, and exhaustive +checksums. It contains no fabricated pair ratios or aggregate case statistics. + +The decision command rejects an existing output directory, creates and owns a +new contained child before reading profiles. Missing +or tampered checksums, incomplete/failed profiles, mixed identity, malformed raw +evidence, and interruption therefore still produce a checksummed failed +`paired-decision.json`, return nonzero, and leave unverified profiles explicitly +missing, invalid, failed, or not evaluated. + +The implemented diagnostic decision layer is read-only. It accepts exactly the +four named artifacts, verifies checksum coverage, strict nested fields, profile +identity, common reference/candidate SHAs and binary hashes, +contract/toolchain/PostgreSQL pins, image digest, fixed pair inventory, +sampler-owned duration, hard state, counters, and cleanup, then applies +precedence: + +```text +contract/governance → functional → correctness/evidence +→ instability → performance → pass +``` + +Stable classifications are `CONTRACT_INVALID`, `PAIR_INVENTORY_INVALID`, +`REFERENCE_GOVERNANCE_INVALID`, `BINARY_IDENTITY_INVALID`, +`EXECUTION_CONTRACT_MISMATCH`, `REFERENCE_FUNCTIONAL_FAILURE`, +`CANDIDATE_FUNCTIONAL_FAILURE`, `CORRECTNESS_REGRESSION`, +`EVIDENCE_INTEGRITY_FAILURE`, `BENCHMARK_ENVIRONMENT_UNSTABLE`, +`PERFORMANCE_REGRESSION`, `CANDIDATE_TIMEOUT_INCONCLUSIVE`, +`CI_INFRASTRUCTURE_TIMEOUT`, `DIAGNOSTIC_TIME_BUDGET_EXCEEDED`, +`DIAGNOSTIC_QUALIFIED`, `DIAGNOSTIC_REJECTED`, and +`PASS`. No performance verdict survives an earlier classification. + +The two `DIAGNOSTIC_*` values are diagnostic-only terminal classifications. +Diagnostic artifacts and decisions cannot emit production `PASS` or carry +production authority. Production sampling and production decisions remain +hard-disabled until a separate governance change supplies and +authorizes the required manifest and threshold policy. + +The per-command safety timeout is fixed at 600 seconds but never exceeds the +remaining internal profile budget. Candidate safety timeout is inconclusive; +reference safety timeout is infrastructure failure. The internal profile +deadline is 35 minutes and a future profile job remains 45 minutes, reserving +five minutes for evidence/cleanup plus five minutes of upload contingency. The +decision workflow timeout remains 10 minutes. + +A future launcher must not expose keys, credentials, usernames, database/DSN +components, generated identifiers, actual host checkout paths, or dynamically +generated repository, build, evidence, and temporary paths through YAML +environment displays. Static GitHub-managed runtime aliases such as +`/github/workspace`, `/github/runner_temp`, `/github/home`, and +`/github/workflow` are allowed platform metadata and do not require retroactive +masking. The launcher must mask actual and generated roots before possible +output, provision its +isolated container after masking, generate and mask ephemeral values with shell +tracing disabled, pass them directly to subprocesses, and always upload the +harness-finalized artifacts. The reusable source audit accepts an explicit +launcher path; no launcher is retained by this correction. + +Bounded-v2 run `30693345495` created each exact sample output child before +invocation, so all four profiles failed `EVIDENCE_INTEGRITY_FAILURE` before +sampling. No fixture execution, paired ratio, MAD, hard-state, counter, cleanup, +or time-budget conclusion exists. The decision path behaved correctly: after +all four profile artifacts were missing, it emitted a checksummed failed +decision without marking missing evidence verified. This run does not qualify +the v2 fixtures. Corrected run `30696834430` supplied the later complete +qualification evidence and produced the functional-architecture acceptance and +hosted-timing rejection recorded above. + +The stopped local A/A probe under `/tmp` is non-authoritative partial evidence: +it completed only the two excluded warmups, has a non-PASS classification, and +does not qualify the fixtures, hosted runner, reference, or thresholds. It is +not an input to this contract or any future decision. + +## Remaining authorization gates + +No additional hosted performance diagnostic is required. Any controlled-runner +hard performance gate, paired production activation, immutable reference +manifest, or numeric threshold policy requires a separate design and +authorization. Phase 11 closure also remains separate from this implementation. + +Phase 11 is **Complete**; its independent closure evidence is recorded in +[the Phase 11 closure evidence](v1.13.11-phase11-closure.md). BKC-016 remains +`Deferred — documented`, and Phase 12 remains unauthorized. diff --git a/docs/release/v1.13/v1.13.11-phase11-repository-coordination-contract.md b/docs/release/v1.13/v1.13.11-phase11-repository-coordination-contract.md new file mode 100644 index 00000000..2e6057f3 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase11-repository-coordination-contract.md @@ -0,0 +1,448 @@ +# Coldkeep v1.13.11 Phase 11 — Repository Coordination Contract + +**Status:** Complete; exact-head CI closure recorded +**Branch:** `release/v1.13.11` + +## Phase boundary + +Phase 11 defines the same-host repository coordination contract without +implementing an operating-system lock. Phase 12 owns native Linux, macOS, and +Windows acquisition and CLI integration. Phase 13 owns independent-process +contention, crash release, restore/live-GC barrier proof, and PostgreSQL +advisory-lock session ownership. + +This phase adds no subprocess contention tests, advisory-lock change, SQLite +live GC, workflow selector, schema, migration, storage-format, codec, or Engine +behavior change. + +## Implemented package inventory + +All seven files in `internal/coordination` use package `coordination`. + +| File | Contract contents | Native acquisition or OS lock imports | Tests | +| --- | --- | --- | --- | +| `contract.go` | Exported `Mode`, `Operation`, `Request`, `Lease`, `Coordinator`, `WithLease`, and `ValidateRequest`; unexported `isCanonicalOperation` | None | `contract_test.go` | +| `identity.go` | Exported `Identity`, `ResolveIdentity`, `ValidateIdentity`, `ControlDirectory`, and fixed artifact-name constants; unexported path-resolution/hash helpers | None; ordinary `os.Lstat` and `filepath.EvalSymlinks` only | `identity_test.go`, `contract_test.go` | +| `errors.go` | Four exported sentinel error variables | None | `owner_test.go`, `contract_test.go`, `identity_test.go` | +| `owner.go` | Exported `Owner`, schema-version constant, `NewOwner`, `EncodeOwner`, `DecodeOwner`, and `ValidateOwner`; unexported hash-format helper | None; ordinary PID, hostname, and executable inspection only | `owner_test.go`, `contract_test.go` | +| `contract_test.go` | Fake lease/coordinator lifecycle, error joining, context, request validation, and fake non-reentrancy | None | Self-contained fake tests | +| `identity_test.go` | Relative, absolute, separator, symlink, missing-leaf, distinct-identity, hash, and control-namespace contracts | None | Self-contained temporary-path tests | +| `owner_test.go` | JSON schema, UTC, optional fields, operation validation, sensitive-data boundary, malformed data, and sentinel discovery | None | Self-contained metadata tests | + +The package imports no `syscall`, `golang.org/x/sys`, `flock`, Windows locking +API, or other native locking package. It opens no lock artifact and contains no +filesystem lock acquisition. + +The implemented internal API shape is: + +```go +type Mode string +const ModeExclusive Mode = "exclusive" + +type Operation string + +type Identity struct { + CanonicalPath string + Hash string +} + +type Owner struct { + SchemaVersion int + PID int + Operation Operation + StartedAt time.Time + Hostname string + Executable string + Version string + IdentityHash string + Mode Mode +} + +type Request struct { + Operation Operation + Mode Mode + Owner Owner +} + +type Lease interface { + Release() error +} + +type Coordinator interface { + Acquire(context.Context, Identity, Request) (Lease, error) +} +``` + +No public Engine API changes. + +## COORD-001 — repository identity and namespace + +The coordination identity is the configured container namespace derived from +`COLDKEEP_STORAGE_DIR`/`container.ContainersDir`. `ResolveIdentity` rejects +empty, NUL-containing, and UNC/network paths; makes local paths absolute and +clean; resolves existing symlinks; and resolves the nearest existing ancestor +when leaf components do not yet exist. Resolution is non-mutating in Phase 11. + +The diagnostic identity is a lowercase SHA-256 hash of the canonical path. +Owner metadata never includes the canonical path, PostgreSQL DSN, credentials, +source path, destination path, user name, or full argument list. + +The future native artifacts belong under: + +```text +/.coldkeep-control/ +``` + +The fixed names are `repository.lock` and `owner.json`. A subdirectory is +required because current startup recovery skips directories but treats every +non-directory entry directly in the container root as a possible orphan +container. + +PostgreSQL aliases do not affect coordination when the container directory is +the same. Different container directories are different identities. Sharing +one PostgreSQL catalog across different container roots is unsupported. Two +databases sharing one container root conservatively share the filesystem +coordination namespace. No repository UUID or schema change is introduced. + +## COORD-002 — operation policy + +The v1.13.11 model is repository-wide, exclusive-only, and fail-fast. Every +participating operation conflicts with every other participating operation. +The pure CLI policy seam classifies: + +- Store and Store Folder; +- by-ID and stored-path Remove and Restore; +- Repair and Garbage Collect, including dry-run; +- Stats, Inspect, List, Search, and all Verify levels; +- Doctor; +- Config get and set; +- snapshot create, delete, restore, list, show, stats, and diff. + +Help, version, `init`, isolated simulation, isolated benchmarks, invalid +commands, and command-level help do not participate. The seam performs no +acquisition. Phase 12 must consume it from one top-level runtime wrapper. + +The exact policy is: + +| CLI input | Phase 11 policy | +| --- | --- | +| `store`, `store-folder`, `restore`, `remove`, `repair`, `gc`, `stats`, `inspect`, `list`, `search`, `verify`, `doctor` | Exclusive, using the matching canonical operation | +| `config get`, `config set` | Exclusive as `config-get` or `config-set` | +| `snapshot create`, `delete`, `restore`, `list`, `show`, `stats`, `diff` | Exclusive, using the matching snapshot operation | +| `init`, `simulate`, `benchmark`, `help`, `-h`, `--help`, `version`, `-v`, `--version` | Bypass | +| Blank or unknown top-level command; missing/unknown config or snapshot subcommand | Zero policy: not required, blank operation and mode | +| Any parsed command with `--help` or `-h` | Bypass | + +`repositoryCoordinationPolicyFor(parsedCommandLine)` is pure, deterministic, +does not open a DB or touch the filesystem, and can run before repository +initialization. It consumes already-parsed state: `parseCommandLine` owns raw +argument behavior. Tests prove help before a command, command-level help, and +config/snapshot subcommands after `--`. There are no coordinated command +aliases beyond the existing help/version bypass spellings, and the seam never +returns a shared mode. + +## COORD-003 — acquisition and errors + +The internal contract exposes only `ModeExclusive`, stable path-free +`Operation` values, `Identity`, `Owner`, `Request`, `Lease`, and `Coordinator`. +Acquisition accepts a context and returns one explicit lease. Phase 12 must +acquire before DB connection/schema work, startup recovery, Engine +construction, transactions, advisory locks, row locks, or filesystem work. + +Stable sentinels are: + +- `ErrRepositoryBusy`; +- `ErrRepositoryLockUnsupported`; +- `ErrRepositoryIdentityInvalid`; +- `ErrNestedRepositoryAcquisition`. + +They are package-level `error` values, not typed errors. Contract code wraps +sentinels with `%w` or combines causes with `errors.Join`, so `errors.Is` +remains available. Phase 11 attaches no owner/conflict-detail type. Malformed +owner JSON returns a metadata parse/validation error; no conflict path exists +yet to consume it. + +Context cancellation and deadlines, `fs.ErrPermission`, and underlying +unexpected I/O causes must remain discoverable through wrapping. Raw OS error +text is not a classification boundary. v1.13.11 adds no wait mode, retry +cadence, timeout flag, or shared lock. + +Only pre-acquisition cancellation/deadline preservation is executable in +Phase 11. Native busy, permission, unsupported-filesystem, lock-corruption, +owner-detail, and raw-OS-error mapping remain Phase 12 obligations and are not +claimed by these sentinels alone. + +## COORD-004 — diagnostic owner metadata + +Owner metadata is versioned JSON containing: + +- schema version; +- PID; +- canonical operation; +- UTC start time; +- optional hostname and executable basename; +- Coldkeep version; +- identity hash; +- exclusive mode. + +Encoding validates all required fields. Decoding rejects unknown fields, +trailing values, invalid hashes, unsupported schema versions, non-positive +PIDs, non-canonical operations, executable paths rather than basenames, and +non-exclusive modes. Encoding and decoding normalize timestamps to UTC. +Hostname and executable are optional and omitted when empty. Metadata is +diagnostic only; native kernel +ownership remains authoritative. Phase 12 must write it atomically after +native acquisition and treat unreadable or stale metadata as unavailable +diagnostics, never as the locking mechanism. + +Phase 11 implements deterministic JSON serialization only. It does not write, +atomically replace, read from, remove, or clean up any metadata file. + +## COORD-005 — lifecycle and cleanup + +`WithLease` validates identity, mode, operation, and matching owner metadata, +then acquires before running the callback. It: + +- does not run the callback when acquisition fails; +- releases after success and every returned error; +- returns a release failure after operation success; +- joins operation and release failures with the operation error first; +- preserves pre-acquisition cancellation and deadline errors; +- releases during panic unwinding without recovering the panic. + +Concrete Phase 12 leases must make `Release` idempotent. Diagnostic metadata +cleanup failure is non-authoritative and must be logged without pretending a +completed operation was rolled back. + +The test lease is fake-only: the first successful release invokes its cleanup +once, a repeated release returns nil, and configured release errors remain +discoverable. It has no native handle, finalizer, or validity probe. +`WithLease` has the exact signature +`WithLease(context.Context, Coordinator, Identity, Request, func() error) +error`. Its order is acquire, callback, release, then error combination. +Acquisition errors bypass the callback. Callback errors remain first when +joined with release errors, and a release error converts callback success into +failure. The helper does not recover panics; its deferred release still runs. +Metadata cleanup is not represented separately. + +## COORD-006 — explicit, non-reentrant ownership + +The top-level repository runtime owns one lease for the complete command or +batch. Engine, storage, snapshot, maintenance, recovery, container, and DB +helpers do not acquire independently. Repeated same-identity acquisition is an +error rather than reference-counted reentrancy. Different identities remain +independent, and no current operation requires two repository leases. + +The Phase 11 fake coordinator proves only the observable non-reentrant contract. +It rejects a repeated normalized hash, including a tested symlink alias, while +allowing a different identity. Release permits later reacquisition. There is +no upgrade behavior, reference counting, package-global production registry, +or production process-local tracking. It is not process-exclusion evidence. + +## COORD-007 — lock ordering + +The required ordering is: + +```text +repository coordination lease +then PostgreSQL advisory lock where applicable +then database transaction +then database row locks +then container or destination filesystem operations +``` + +Repository coordination does not replace Phase 10 row locks, PostgreSQL +NOWAIT/SKIP LOCKED behavior, SQLite transaction behavior, restore pins, or +live-GC advisory locking. It does not make database/filesystem operations +atomic. + +The current live-GC advisory acquire and release paths use pooled `*sql.DB` +calls even though PostgreSQL advisory ownership is session-scoped. That +unchanged concern remains Phase-13-owned and must use one dedicated +`*sql.Conn` before live-GC contention is considered proven. + +## COORD-008 — Restore, verification, and GC + +Restore uses exclusive repository coordination for planning, pinning, +container reads, destination writes, metadata application, reader cleanup, and +unpin. Existing pins remain required but do not cover pre-pin planning, +snapshot membership, startup recovery, or all container races. + +Fast, standard, full, and deep verification all use exclusive coordination. +This avoids reporting a mixed database/container view while Store, recovery, +Remove, or live GC changes repository state. + +GC dry-run also uses exclusive coordination because its plan spans +reachability, pins, metadata, and container files. PostgreSQL live GC uses the +repository lease as the outer barrier and retains advisory and row locks. +SQLite live GC remains explicitly unsupported. + +## COORD-009 — filesystem support boundary + +The intended guarantee is same-host coordination on supported local +filesystems using native Linux, macOS, and Windows implementations. It excludes +cross-host, distributed, NFS, SMB, cloud-synchronized, and arbitrary +mount-alias guarantees. Same-host container bind mounts qualify only when all +processes reach the same local backing lock artifact. + +External deletion or replacement of an active control directory is +unsupported. Read operations will require write permission for the control +namespace after Phase 12 integration. + +## COORD-010 — proof separation + +Phase 11 contract tests cover: + +- path normalization and safe identity hashing; +- recovery-safe control-directory naming; +- CLI operation-to-exclusive-mode classification; +- explicit bypass commands; +- owner JSON round trips and sensitive-field absence; +- acquisition/callback/release ordering; +- cancellation, deadlines, release failure, and joined errors; +- nested same-identity refusal and separate-identity independence. + +These tests do not prove an OS lock, process exclusion, crash recovery, native +platform behavior, or live-GC barriers. Phase 12 owns native unit and +cross-platform evidence. Phase 13 owns subprocess and PostgreSQL live-GC +contention evidence. + +The new top-level tests are: + +- lifecycle/errors: `TestWithLeaseLifecycle`, + `TestWithLeaseDoesNotRunAfterAcquireFailure`, + `TestWithLeaseCombinesOperationAndReleaseErrors`, + `TestWithLeaseReturnsOperationErrorAfterReleaseSuccess`, + `TestWithLeaseReturnsReleaseErrorAfterOperationSuccess`, + `TestLeaseContractReleaseIsIdempotentAfterSuccess`, + `TestWithLeasePreservesContextCancellation`, + `TestWithLeasePreservesExpiredDeadline`, + `TestWithLeaseRejectsUnsupportedModeAndInvalidInputs`, + `TestValidateRequestRejectsMismatchedOwner`, and + `TestCoordinationValidateRequestRejectsBlankAndUnknownOperations`; +- nested acquisition: `TestCoordinatorContractRejectsNestedIdentityAndSeparatesRepositories` + and `TestCoordinatorContractTreatsAliasAsNestedIdentity`; +- identity: `TestResolveIdentityNormalizesAliasesAndTrailingSeparators`, + `TestResolveIdentityNormalizesRelativeAndAbsoluteForms`, + `TestResolveIdentityUsesResolvedNearestExistingAncestor`, + `TestResolveIdentityRejectsInvalidAndNetworkPaths`, + `TestIdentityResolutionPreservesUnderlyingFilesystemCause`, + `TestIdentityHashIsStableAndPathFree`, + `TestResolveIdentitySeparatesDistinctRepositories`, + `TestCoordinationControlDirectoryUsesRecoverySafeSubdirectory`, and + `TestValidateIdentityRejectsTampering`; +- metadata/errors: `TestOwnerMetadataRoundTripAndSensitiveFieldBoundary`, + `TestOwnerMetadataOptionalFieldsAndExecutableBasename`, + `TestOwnerMetadataRejectsUnknownOperation`, + `TestOwnerMetadataRejectsMalformedAndUnknownFields`, and + `TestCoordinationStableErrorSentinelsRemainDiscoverable`; +- CLI policy: `TestRepositoryCoordinationPolicyRequiresExclusiveRepositoryOperations`, + `TestRepositoryCoordinationPolicyBypassesNonRepositoryCommands`, + `TestRepositoryCoordinationPolicyUsesParsedHelpAndDoubleDashSemantics`, and + `TestRepositoryCoordinationPolicyBypassesCommandHelp`. + +They spawn no subprocess, call no OS lock API, require no PostgreSQL or network +access, and use no `t.Parallel`; race and ten-repeat profiles pass. + +## Phase 12 and Phase 13 handoff + +Phase 12 still owns native Linux, macOS, and Windows fail-fast locking, +creation and final re-resolution of `.coldkeep-control`, native-artifact alias +behavior, same-process non-reentrancy, atomic owner-metadata persistence, +stale/corrupt diagnostic handling, busy/unsupported/permission mapping, the +common CLI wrapper before DB connection and recovery, and runtime release +ordering. Phase 11 supplies constants, pure normalization, interfaces, +`WithLease`, comments, and fake tests only. + +Phase 13 still owns independent-process contention, normal-exit release, +killed-process release, stale metadata non-blocking behavior, distinct +repository process isolation, Restore/live-GC and mutation/live-GC exclusion, +one dedicated PostgreSQL advisory-lock `*sql.Conn`, corruption checks after +contention or killed holders, and continued SQLite live-GC refusal. No Phase 13 +test or production change exists. + +## BKC boundary + +- BKC-014 remains `Backend-specific — proven`. +- BKC-015 remains `Backend-specific — proven` within Phase 10's database-local + scope. +- BKC-016 remains `Deferred — documented`: the coordination contract is + defined, but native implementation and contention proof are pending. + +After Phases 12 and 13, the intended bounded BKC-016 classification is +`Platform-specific — proven` for fail-fast exclusive same-host coordination on +supported Linux, macOS, and Windows local filesystems. Cross-host and +network-filesystem claims remain excluded. + +## Local validation + +All planned local Phase 11 checks passed: + +| Validation | Result | +| --- | --- | +| Focused verbose coordination and CLI policy contracts | PASS; coordination `0.012s`, CLI `0.020s`, wall `4.044s` | +| Ten-repeat focused contract profile | PASS; coordination `0.056s`, CLI `0.057s`, wall `1.170s` | +| Focused race profile | PASS; coordination `1.059s`, CLI `1.064s`, wall `6.771s` | +| Full affected-package race profile | PASS; coordination `1.044s`, CLI `1.787s`, wall `2.910s` | +| Affected Engine, storage, snapshot, maintenance, container, DB, and recovery packages | PASS; wall `9.850s` | +| Full `go test ./...` repository suite | PASS; wall `36.830s` | +| Linux `GOOS=linux` no-test execution | PASS; package `0.004s` | +| macOS compile-only coordination test binary | PASS; Mach-O 64-bit x86-64 | +| Windows compile-only coordination test binary | PASS; PE32+ x86-64 | +| `go vet ./...` | PASS | +| PATH `golangci-lint` v2.11.3 | PASS; zero issues | +| CI-pinned `golangci-lint` v2.6.2 | PASS; zero issues | +| Release-state validator unit suite | PASS; 48 tests in `5.787s` | +| Release-state validator real repository | PASS; development state, zero violations | +| Smart quotes, validation matrix, versioned row writers, local CI audit, and `git diff --check` | PASS | + +Linux tests executed with `-run '^$'`; macOS and Windows packages were compiled +with `go test -c` and were not executed on the Linux host. No build tags are +required. This does not claim native locking. Go and lint caches were placed +under `/tmp` because the environment's default cache paths are read-only. + +The exact release-state JSON is: + +```json +{"status":"ok","validator":"coldkeep-release-state","state":"development","active_version":"1.13.11","violations":[],"error":null} +``` + +Exact-head remote CI acceptance is recorded in +[the Phase 11 closure evidence](v1.13.11-phase11-closure.md). No +PostgreSQL-specific event is required for this contract-only phase. + +## Changed-file boundary + +Production contract and policy: + +- `internal/coordination/contract.go` +- `internal/coordination/identity.go` +- `internal/coordination/errors.go` +- `internal/coordination/owner.go` +- `cmd/coldkeep/repository_coordination_contract.go` + +Contract tests: + +- `internal/coordination/contract_test.go` +- `internal/coordination/identity_test.go` +- `internal/coordination/owner_test.go` +- `cmd/coldkeep/repository_coordination_contract_test.go` + +Release evidence consists of this report, `CHANGELOG.md`, and the seven +canonical v1.13.11 tracker/matrix documents. No Phase 12 lock implementation or +Phase 13 proof file is included. + +The coordination-contract commit changed none of `cmd/coldkeep/main.go`, +workflows, Engine, storage, snapshot, maintenance, container, DB, recovery, +schemas, migrations, storage formats, codecs, PostgreSQL advisory locking, +SQLite live GC, startup recovery routing, or DB connection order. + +Subsequent exact-head CI exposed an unrelated benchmark-gate integrity blocker. +The bounded diagnostic bootstrap changes only benchmark command/reporting +code, validation scripts, and a manual workflow; it does not alter repository +runtime behavior or broaden the coordination claim. See +[v1.13.11-phase11-benchmark-gate-integrity-remediation.md](v1.13.11-phase11-benchmark-gate-integrity-remediation.md). +The historical calibration blocker is retained as evidence of the boundary that +was corrected before acceptance. Phase 11 closure is recorded at the accepted +exact head in [the closure evidence](v1.13.11-phase11-closure.md); it does not +authorize Phase 12 implementation. diff --git a/docs/release/v1.13/v1.13.11-phase12-closure.md b/docs/release/v1.13/v1.13.11-phase12-closure.md new file mode 100644 index 00000000..87137ccc --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase12-closure.md @@ -0,0 +1,239 @@ +# Coldkeep v1.13.11 Phase 12 — Repository Coordination Runtime Closure + +**Phase:** 12 — Cross-Platform Repository Lock Implementation +**Status:** Complete +**Closure authority:** documentation and evidence only +**Accepted implementation head:** `cf75530524aebd035d17b17ac7e3765a42f32cf8` + +## Closure decision + +Phase 12 is complete. The Phase 11 coordination contract is implemented as a +repository-wide, exclusive, fail-fast, non-reentrant Lease for same-host local +filesystems. Native runtime and production Coordinator lifecycle are proven on +Linux, macOS, and Windows. Stable CLI coordination errors, top-level runtime +ordering, direct-caller responsibilities, and the Restore/Verify/GC outer-Lease +policy are implemented and tested. + +Phase 13A was completed early as a bounded tests-only prerequisite after Phase +12G exposed a historical adversarial-test assumption that conflicted with the +newly enforced Lease. It did not correct or weaken production coordination. + +The release remains active and incomplete. Phase 13 is Next, with Phase 13B +owning killed-holder release and reacquisition proof. BKC-016 remains +**Deferred — documented**. + +## Frozen coordination scope and hierarchy + +The closed contract remains: + +- repository-wide and exclusive-only; +- fail-fast and non-reentrant; +- same-host and local-filesystem scoped; and +- identified by the canonical container/storage namespace. + +The required acquisition hierarchy is: + +```text +repository Lease +then database/schema work +then existing database locks and transactions +then filesystem work +``` + +The Lease is not distributed coordination and does not make different +container namespaces sharing one database safe. + +## Phase 12 slice reconciliation + +| Slice | Commit | Closed implementation boundary | +| --- | --- | --- | +| 12A | `b63bd343eee990280bbe7912dd254e76f1ed3793` | Prepared `.coldkeep-control`, safe creation modes and path validation, and final identity re-resolution | +| 12B | `e13db339aff44e668beff5647ec348cb725f1779` | Bounded diagnostic owner metadata and identity-keyed process reservation | +| 12C | `ac1a6bb0a690710b8130ec9036646d366b3f9f2a` | Linux/Darwin nonblocking exclusive `flock` lifecycle | +| 12D | `bbe2e184a820196eb060c3dbcd28c01a9edc136d` | Windows nonblocking exclusive `LockFileEx` lifecycle and reparse rejection | +| 12E | `5b8f749339287fcb9de82ffc9b90a7addff24bae` | Production Coordinator, CLI lifecycle ordering, and bounded file-backed output spooling | +| 12F | `9934bc9a1f6db64e7c80e8893b0bcb2d95f93d6a` | Stable error surface, direct-caller contract, and Restore/Verify/GC coverage | +| 12G | `6a36804838273b3b7aeb1d2febff17dbe29c2f78` | Required cross-platform native and production Coordinator runtime CI | + +The detailed Phase 11 contract remains in +[the repository coordination contract](v1.13.11-phase11-repository-coordination-contract.md). +Phase 12G runtime evidence is recorded in +[the cross-platform native runtime report](v1.13.11-phase12g-cross-platform-native-runtime-ci.md). + +## Implementation chronology + +Phase 12A established the safe control namespace. Phase 12B added diagnostic +metadata and same-process ownership. Phases 12C and 12D added Unix and Windows +native locking. Phase 12E assembled those layers into the production +Coordinator and acquired one outer CLI Lease before recovery and database +work. Phase 12F froze public error classification and direct-call boundaries. +Phase 12G moved the native and production Coordinator tests into the required +Linux, macOS, and Windows matrix. + +No Phase 12H production, CLI, test, workflow, schema, storage-format, or module +change is required. + +## Phase 13A dependency inversion + +The first Phase 12G CI run proved every selected native runtime but failed the +aggregate Required Gate. Two historical G6 tests expected overlapping real CLI +stores to succeed, which contradicts the repository-wide exclusive fail-fast +Lease. + +Phase 13A therefore occurred before Phase 12 closure. Commit +`cf75530524aebd035d17b17ac7e3765a42f32cf8` changed only +`tests/adversarial/g6_concurrent_operations_adversarial_test.go`. It added a +deterministic production-Lease holder protocol, aligned overlap expectations +with `REPOSITORY_BUSY`, serialized the integrity scenarios, and strengthened +the shared-chunk fixture. It changed no production coordination, CLI, +native-lock, workflow, backend, or public API behavior. Detailed evidence is +recorded in +[the Phase 13A reconciliation report](v1.13.11-phase13a-process-contention-reconciliation.md). + +## Local validation evidence + +At the Phase 13A implementation state, focused contention and integrity tests +passed with and without the race detector, the synchronized contention proof +passed twenty consecutive repetitions, and the complete PostgreSQL adversarial +suite passed under the race detector. Repository-wide normal and race tests, +`go vet ./...`, module checks, and diff hygiene also passed as recorded in the +Phase 13A evidence. + +Phase 12H changes documentation only. Its bounded local gate consists of diff +and module hygiene, release-state validator tests and real-state validation, +the validation-matrix audit, and the local CI-enforcement audit. + +## Initial Phase 12G hosted evidence + +Initial CI run `31255838594`, attempt 1, is preserved as a failed aggregate +run: + +| Evidence | Job | Result | +| --- | --- | --- | +| Linux native | `93098935573` | Success | +| macOS native | `93098935561` | Success | +| Windows native | `93098935567` | Success | +| Required Gate | `93100650890` | Failed | +| Independent CodeQL run | `31255838579` | Success | + +The native jobs are valid runtime proof. The Required Gate failed because of +the stale G6 successful-overlap expectation, so this run is not described as +globally successful. + +## Successful reconciliation hosted evidence + +CI run `31258313120`, attempt 1, passed at the Phase 13A implementation commit: + +| Evidence | Job | Result | +| --- | --- | --- | +| Adversarial plain | `93106243492` | Success | +| Adversarial AES-GCM | `93106243503` | Success | +| Linux native | `93105065213` | Success | +| macOS native | `93105065193` | Success | +| Windows native | `93105065187` | Success | +| Required Gate | `93106823979` | Success | +| Independent CodeQL run | `31258313118` | Success | + +All selected native and production Coordinator tests executed. None skipped. +The Windows symlink/reparse rejection test executed and passed. + +## Cross-platform native and Coordinator proof + +- **Linux native runtime:** PROVEN +- **macOS native runtime:** PROVEN +- **Windows native runtime:** PROVEN +- **Production Coordinator lifecycle on Linux/macOS/Windows:** PROVEN +- **Same-process non-reentrancy and successor protection:** PROVEN +- **Linux independent-process fail-fast contention:** PROVEN +- **macOS/Windows independent-process contention:** NOT PROVEN + +The cross-platform matrix proves actual Darwin `flock` and Windows +`LockFileEx` acquisition, contention, release, artifact persistence, and +production Coordinator acquisition/reacquisition on same-host local +filesystems. + +## Stable errors, direct callers, and runtime operations + +The closed CLI mapping is: + +| Cause | Public code | Exit | +| --- | --- | --- | +| `ErrRepositoryBusy` | `REPOSITORY_BUSY` | 1 | +| `ErrRepositoryLockUnsupported` | `REPOSITORY_LOCK_UNSUPPORTED` | 1 | +| `ErrRepositoryIdentityInvalid` | `INVALID_ARGUMENT` | 2 | +| `ErrNestedRepositoryAcquisition` | `INTERNAL` | 1 | +| permission | `PERMISSION_DENIED` | 1 | +| `context.Canceled` | `CANCELED` | 1 | +| `context.DeadlineExceeded` | `DEADLINE_EXCEEDED` | 1 | +| unexpected coordination I/O | `INTERNAL` | 1 | + +Wrapped and joined causes remain discoverable through `errors.Is`, with the +operation error retaining precedence when operation and release both fail. + +A direct library caller operating on a real or shared repository must acquire +the Lease before database/schema/Engine setup, retain it through operation and +runtime cleanup, and release afterward. Engine methods intentionally do not +acquire a hidden Lease. Isolated single-owner temporary fixtures do not +constitute coordination proof. + +Restore, every Verify level, GC dry-run, and live GC receive an exclusive outer +Lease. This closes CLI routing and lifecycle coverage; it does not prove the +later live-GC cross-process barrier. + +## G6 integrity preservation + +Phase 13A retained six independent real-CLI same-file stores, exact graph +convergence, full verification, metadata invariants, healthy restore/hash +checks, PostgreSQL execution, and plain/AES-GCM coverage. The shared-input +fixture now uses a deterministic 2 MiB common prefix and requires an actual +shared `chunk_id`. The dedicated contention test replaces only the obsolete +successful-overlap claim. + +## Local stale-path diagnostic + +The separately observed +`TestSchemaStartupOperatorMessagingReleaseGate/missing_schema_message` failure +is classified as a **non-blocking local fixture artifact**. The subtest +inherited the mutable package-global `container.ContainersDir` after it pointed +at a deleted temporary directory. The production Coordinator correctly +rejected that stale repository identity before the expected schema error. + +This is not Phase 12 product-failure evidence. The hosted correctness, +integration-stress, and long-run jobs passed at the implementation commit, and +the aggregate Required Gate was green. No out-of-scope fixture correction is +included in Phase 12H. + +## Proof boundary and Phase 13 deferrals + +- **Killed-process native release/reacquisition:** NOT PROVEN / PHASE 13B +- **Live-GC cross-process barrier:** NOT PROVEN / LATER PHASE 13 +- **PostgreSQL advisory locking on one dedicated `*sql.Conn`:** NOT CLOSED / LATER PHASE 13 +- **Cross-host coordination:** NOT SUPPORTED +- **Distributed coordination:** NOT CLAIMED +- **Network-filesystem safety:** NOT GUARANTEED +- **FreeBSD:** COMPILE-ONLY UNSUPPORTED BACKEND + +The helper's emergency process-kill cleanup is not killed-holder semantic +evidence. + +## Benchmark governance and BKC-016 + +Benchmark governance remains frozen: GitHub-hosted benchmark integrity is hard +required, hosted timing is advisory, and hard timing-regression enforcement is +deferred to controlled infrastructure. + +BKC-016 remains **Deferred — documented**. Phase 12 closes the native runtime +and bounded lifecycle work, but the remaining Phase 13 proof prevents a broader +classification. + +## Final phase state + +- **Phase 12A–12G:** COMPLETE +- **Phase 12H:** COMPLETE — documentation/evidence closure only +- **Phase 13A:** COMPLETE — bounded early prerequisite +- **Phase 13:** NEXT +- **Phase 13B:** NOT STARTED +- **BKC-016:** Deferred — documented +- **v1.13.11 release:** Active / incomplete + +**Phase 12 closure verdict:** READY / COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase12g-cross-platform-native-runtime-ci.md b/docs/release/v1.13/v1.13.11-phase12g-cross-platform-native-runtime-ci.md new file mode 100644 index 00000000..94ef87d8 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase12g-cross-platform-native-runtime-ci.md @@ -0,0 +1,88 @@ +# Coldkeep v1.13.11 Phase 12G — Cross-Platform Native Runtime CI Evidence + +**Phase:** 12G — Cross-platform native repository coordination runtime proof + +**Status:** Complete after Phase 13A reconciliation + +**Implementation commit:** `6a36804838273b3b7aeb1d2febff17dbe29c2f78` + +**Reconciliation commit:** `cf75530524aebd035d17b17ac7e3765a42f32cf8` + +## Evidence history + +Phase 12G required two hosted observations. The first run proved the native +runtime matrix but failed the Required Gate because historical G6 tests still +expected overlapping repository-wide CLI stores to succeed. Phase 13A aligned +those tests with the frozen exclusive, fail-fast coordination contract. The +second run preserved the native proof and completed the Required Gate. + +| Evidence | Result | +| --- | --- | +| Initial CI run `31255838594`, attempt 1 | Failed | +| Linux job `93098935573` | Success | +| macOS job `93098935561` | Success | +| Windows job `93098935567` | Success | +| Initial Required Gate job `93100650890` | Failed | +| Initial failure cause | Historical G6 successful-overlap expectation | +| Initial CodeQL run `31255838579` | Success | +| Reconciliation CI run `31258313120`, attempt 1 | Success | +| Reconciliation Required Gate job `93106823979` | Success | +| Reconciliation CodeQL run `31258313118`, attempt 1 | Success | + +The initial run is not classified as globally successful. Its three native +jobs remain valid native-runtime evidence, while its Required Gate result +remains failed. + +## Reconciliation native matrix + +Run `31258313120` executed this exact blocking command in every existing +cross-platform matrix cell: + +```bash +go test -v -count=1 -run '^(TestNativeLock|TestWindowsNativeLock|TestProductionCoordinator)' ./internal/coordination +``` + +| Runner | Job | Result | +| --- | --- | --- | +| `ubuntu-latest` | `93105065213` | Success | +| `macos-latest` | `93105065193` | Success | +| `windows-latest` | `93105065187` | Success | + +Linux and macOS each ran and passed the three production Coordinator tests and +the Unix artifact, preservation, unsafe-artifact, contention/reacquisition, +stale/idempotent release, different-repository, and 32-contender tests. This +includes real Darwin `flock` acquisition and release. + +Windows ran and passed the three production Coordinator tests and every +selected Windows native test, including persistent artifact creation, existing +and zero-length content preservation, directory and symlink rejection, +contention/reacquisition, stale/idempotent release, different repositories, +32-contender contention, and error mapping. The symlink test passed; there was +no Windows symlink privilege/configuration skip. + +No selected native or production Coordinator test skipped or failed in any +reconciliation matrix cell. + +## Accepted proof boundary + +- **Linux native runtime:** PROVEN +- **macOS native runtime:** PROVEN +- **Windows native runtime:** PROVEN +- **Production Coordinator lifecycle on supported native OSes:** PROVEN +- **Independent-process contention:** PROVEN on Linux by Phase 13A +- **Killed-process release:** NOT PROVEN / PHASE 13B +- **Live-GC cross-process barrier:** NOT PROVEN / LATER PHASE 13 +- **PostgreSQL advisory lock on one dedicated `*sql.Conn`:** NOT PROVEN / LATER PHASE 13 +- **Network/cross-host coordination:** NOT CLAIMED +- **Network-filesystem safety:** NOT GUARANTEED +- **FreeBSD:** COMPILE-ONLY UNSUPPORTED BACKEND + +The native tests use same-host local filesystems. The matrix does not establish +cross-host or network-filesystem locking guarantees. + +## Closure decision + +Phase 12G is complete because the native Linux, macOS, and Windows cells, both +Phase 13A adversarial cells, all pre-existing required jobs, and the Required +Gate passed together in reconciliation run `31258313120`. Phase 12H may use +this evidence but is not started by this document. diff --git a/docs/release/v1.13/v1.13.11-phase13-closure.md b/docs/release/v1.13/v1.13.11-phase13-closure.md new file mode 100644 index 00000000..76de1d60 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase13-closure.md @@ -0,0 +1,194 @@ +# Coldkeep v1.13.11 Phase 13 — Multi-Process Coordination Closure + +**Phase:** 13 — Multi-Process Contention and Live-GC Barrier Proof +**Status:** Complete +**Closure authority:** documentation and evidence only +**Accepted evidence head:** `d4b9caeebc277cafcd9d07e2cf9ca63dde215307` + +## Closure decision + +Phase 13 is complete. Phase 13A proves representative independent-process +contention on Linux, Phase 13B proves kernel-owned release after a killed +holder and immediate reacquisition, and Phase 13C proves real live-GC +cross-process exclusion and correct PostgreSQL advisory-lock session ownership. + +The release remains active and incomplete. Phase 14 — Container Range and +Header Consistency Hardening — is Next. No PR, merge, tag, publication, or +release-branch deletion is authorized by this closure. + +## Frozen repository-coordination contract + +The Phase 11/12 contract remains: + +- repository-wide, exclusive-only, fail-fast, and non-reentrant; +- identified by the canonical container/storage namespace; +- scoped to same-host supported local filesystems; and +- acquired before recovery, database/schema setup, existing database locks, + transactions, and filesystem work. + +Native runtime and production Coordinator lifecycle are proven on Linux, +macOS, and Windows. Owner metadata remains diagnostic only; the native kernel +lock is authoritative. Direct library callers operating on a real or shared +repository must acquire the Lease explicitly. Engine methods do not acquire a +hidden Lease. + +## Phase 13A — independent-process contention and G6 reconciliation + +Implementation `cf75530524aebd035d17b17ac7e3765a42f32cf8` added a +deterministic helper-process holder using the production Coordinator. A real +CLI contender receives the complete `REPOSITORY_BUSY`/exit-1 contract while +the holder owns the Lease, writes no logical-file row, and succeeds after the +explicit `READY → RELEASE → RELEASED` lifecycle. + +The same change reconciled the historical G6 successful-overlap assumption +without weakening production coordination. It retained sequential independent +real-CLI same-file convergence and strengthened the shared-input fixture to a +real deterministic 2 MiB common chunk with an explicit shared-reference +assertion. Repository verification and restore/hash checks remain required in +plain and AES-GCM coverage. + +Detailed evidence is recorded in +[the Phase 13A reconciliation report](v1.13.11-phase13a-process-contention-reconciliation.md). + +## Phase 13B — killed-holder release + +Implementation `5bdaa44324d02b89ca97159cbdc22e2e46a459fe` proves this Linux +transition: + +```text +production Lease held +→ real CLI REPOSITORY_BUSY +→ Process.Kill / SIGKILL +→ bounded Cmd.Wait reaps the holder +→ stale owner.json remains diagnostic only +→ first post-Wait real CLI acquisition succeeds +→ repository verification and restore/hash checks pass +``` + +The semantic path sends no release signal, so application `Lease.Release` +cannot execute. Reacquisition uses no sleep, retry, PID-liveness check, stale +metadata deletion, or force-unlock behavior. `repository.lock` persists. + +Commit `4aa2e75441d8f9f7f6624710b5afeccb9417be54` changed only the +Staticcheck `QF1008` selector from `exitErr.ProcessState.Success()` to the +promoted `exitErr.Success()`. It changed no production or test semantics. + +Detailed evidence is recorded in +[the Phase 13B killed-holder report](v1.13.11-phase13b-killed-process-release.md). + +## Phase 13C — real live-GC cross-process barrier + +Implementation `54b1a04cdd1680f4b173d6ff9cd0c93adeca676f` starts real +`coldkeep gc --output json` against PostgreSQL. A parent transaction holds +`physical_file` in `ACCESS EXCLUSIVE` mode, and `pg_stat_activity`, +`pg_blocking_pids`, and `wait_event_type=Lock` prove that GC reached its +physical-graph preflight while holding the outer repository Lease. + +A real independent Store then receives the complete Busy contract and writes +no row. Releasing the relation barrier lets live GC finish successfully with +`dry_run=false`. The immediate successor Store, repository invariants, and +retained/successor restore SHA-256 checks pass. + +Detailed evidence is recorded in +[the Phase 13C live-GC report](v1.13.11-phase13c-live-gc-coordination.md). + +## PostgreSQL dedicated advisory session + +Implementation `da78614af92835b78ebdf0ac84d94fd1f828f41d` reserves one +dedicated `*sql.Conn`, acquires `pg_try_advisory_lock(847362)` on it, and keeps +that same session checked out throughout GC preflight, planning, sweep, and +cleanup. Unlock executes on the owning session and must return `true` before +the connection returns to the pool. + +Unlock SQL errors and unexpected false results are returned as cleanup errors. +An uncertain lock-owning session is discarded with `driver.ErrBadConn` rather +than returned to the reusable pool. `errors.Join` preserves simultaneous +operation and cleanup failures. PostgreSQL GC fails fast when the pool permits +only one open connection. + +Required tests prove acquisition/unlock backend-PID equality, held-lock +exclusion, immediate independent reacquisition after success and operation +failure, physical-session discard after both cleanup-failure forms, and the +single-connection boundary. SQLite live GC remains explicitly unsupported; +SQLite dry-run remains supported. + +## Implementation and evidence chronology + +| Purpose | Commit | +| --- | --- | +| Phase 13A implementation | `cf75530524aebd035d17b17ac7e3765a42f32cf8` | +| Phase 12G/13A evidence | `eba01b22980331fa33541db360afcc143a8d9ea5` | +| Intervening Phase 12 closure | `c1b5e4d0792e271efb3dfaff6f800590703bf41b` | +| Phase 13B implementation | `5bdaa44324d02b89ca97159cbdc22e2e46a459fe` | +| Phase 13B Staticcheck correction | `4aa2e75441d8f9f7f6624710b5afeccb9417be54` | +| Phase 13B evidence | `701843243487bc9566df213ccfa0c070d44fa7de` | +| Phase 13C advisory correction | `da78614af92835b78ebdf0ac84d94fd1f828f41d` | +| Phase 13C process proof | `54b1a04cdd1680f4b173d6ff9cd0c93adeca676f` | +| Phase 13C evidence | `d4b9caeebc277cafcd9d07e2cf9ca63dde215307` | + +## Hosted evidence chronology + +| Phase | Evidence | Result | +| --- | --- | --- | +| 13A | CI `31258313120`; adversarial jobs `93106243492`/`93106243503`; native jobs `93105065213`/`93105065193`/`93105065187`; Required Gate `93106823979` | Success | +| 13A | CodeQL `31258313118` | Success | +| 13B | CI `31308921622`; adversarial jobs `93234838004`/`93234838028`; native jobs `93233617799`/`93233617752`/`93233617794`; Required Gate `93235437532` | Success | +| 13B | CodeQL `31308921567` | Success | +| 13C implementation | CI `31311628561`; correctness plain `93240210968`; adversarial jobs `93241436681`/`93241436683`; Required Gate `93242095156` | Success | +| 13C implementation | CodeQL `31311628559` | Success | +| 13C evidence head | CI `31312520061`; adversarial jobs `93243740000`/`93243740002`; Required Gate `93244412826` | Success | +| 13C evidence head | CodeQL `31312520060`; Aggregate `93242579705` | Success | + +## Cross-platform proof matrix + +| Evidence | Linux | macOS | Windows | +| --- | --- | --- | --- | +| Native primitive runtime | Proven: `flock` | Proven: Darwin `flock` | Proven: `LockFileEx` | +| Production Coordinator lifecycle | Proven | Proven | Proven | +| Generic independent-process contention | Proven / Phase 13A | Not separately proven | Not separately proven | +| Killed-holder release/reacquisition | Proven / Phase 13B | Not separately proven | Not separately proven | +| Real live-GC cross-process exclusion | Proven / Phase 13C | Not separately proven | Not separately proven | +| PostgreSQL advisory-session ownership | Proven in required PostgreSQL/Linux CI | Not separately executed | Not separately executed | + +The supported-platform guarantee is backed by real native and production +Coordinator runtime proof on all three OSes plus representative +independent-process semantics on required Linux. v1.13.11 does not require or +claim separate macOS/Windows subprocess execution. + +## Public guarantee and proof boundary + +- **Linux/macOS/Windows native runtime:** PROVEN +- **Production Coordinator lifecycle on Linux/macOS/Windows:** PROVEN +- **Linux independent-process contention:** PROVEN +- **Linux killed-holder release:** PROVEN +- **Linux real live-GC cross-process exclusion:** PROVEN +- **PostgreSQL advisory same-session ownership:** PROVEN / CLOSED +- **macOS/Windows independent-process, killed-holder, and live-GC subprocess semantics:** NOT SEPARATELY PROVEN +- **Cross-host/distributed coordination:** NOT CLAIMED +- **NFS, SMB, cloud-synchronized, and arbitrary network-filesystem safety:** NOT GUARANTEED +- **FreeBSD:** COMPILE-ONLY / UNSUPPORTED BACKEND + +PostgreSQL advisory locking remains an inner live-GC singleton mechanism. It +does not create a cross-host repository-coordination guarantee. + +## BKC-016 and benchmark governance + +BKC-016 remains **Deferred — documented**. Phase 13 closure does not promote +or rewrite that claim-matrix classification. + +GitHub-hosted benchmark integrity remains hard required. Hosted timing remains +advisory, and hard timing-regression enforcement remains deferred to +controlled infrastructure. + +## Final phase state + +- **Phase 12A–12H:** COMPLETE +- **Phase 13A:** COMPLETE +- **Phase 13B:** COMPLETE +- **Phase 13C:** COMPLETE +- **Phase 13:** COMPLETE +- **Phase 14:** NEXT +- **BKC-016:** Deferred — documented +- **v1.13.11 release:** Active / incomplete + +**Phase 13 closure verdict:** READY / COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase13a-process-contention-reconciliation.md b/docs/release/v1.13/v1.13.11-phase13a-process-contention-reconciliation.md new file mode 100644 index 00000000..e169b6db --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase13a-process-contention-reconciliation.md @@ -0,0 +1,123 @@ +# Coldkeep v1.13.11 Phase 13A — Process Contention Reconciliation Evidence + +**Phase:** 13A — G6 independent-process coordination reconciliation + +**Status:** Complete + +**Implementation commit:** `cf75530524aebd035d17b17ac7e3765a42f32cf8` + +**Implementation scope:** Adversarial tests only + +## Implemented reconciliation + +Phase 13A changed only +`tests/adversarial/g6_concurrent_operations_adversarial_test.go` in the +implementation commit. It made no production, CLI, native-lock, workflow, +backend, or public API change. + +The implementation: + +- replaced probabilistic same-file CLI overlap with six sequential, + independent real-CLI stores while preserving graph equality, repository + verification, metadata invariants, and restore/hash checks; +- replaced the undersized shared-input fixture with an identical deterministic + 2 MiB prefix plus distinct tails, then required an actual shared `chunk_id` + before verification and restore checks; +- added a separate helper-process holder using the production Coordinator and + a `READY`/`RELEASE`/`RELEASED` stdin/stdout protocol; +- asserted the stable `REPOSITORY_BUSY` JSON contract, exit code 1, empty + success output, and zero logical-file rows while the Lease was held; and +- proved successful store, verification, release, reacquisition, and restore + after the holder released the Lease. + +No sleep establishes ownership. Timeouts only bound stuck harness cleanup. + +## Local validation + +The synchronized independent-process contention test passed for both inner +codecs and passed twenty consecutive repetitions without `-race`. The focused +three-test set passed with and without the race detector. The complete +PostgreSQL adversarial suite passed under the race detector in approximately +319 seconds. + +The following additional checks passed: + +- `git diff --check` +- `go mod verify` +- `go mod tidy -diff` +- `go test -count=1 ./...` +- `go test -race -count=1 ./...` +- `go vet ./...` + +A separate local, DB-enabled integration diagnostic exposed an existing +fixture-order failure in +`TestSchemaStartupOperatorMessagingReleaseGate/missing_schema_message`: the +production Coordinator rejected a stale/deleted container path before the +test's expected schema error. It was outside the authorized Phase 13A file and +was not changed. Hosted correctness, integration-stress, and long-run jobs all +passed at the implementation commit. + +## Hosted reconciliation + +| Evidence | Result | +| --- | --- | +| CI run `31258313120`, attempt 1 | Success | +| `adversarial (plain)` job `93106243492` | Success | +| `adversarial (aes-gcm)` job `93106243503` | Success | +| Linux native job `93105065213` | Success | +| macOS native job `93105065193` | Success | +| Windows native job `93105065187` | Success | +| Required Gate job `93106823979` | Success | +| CodeQL run `31258313118`, attempt 1 | Success | + +Both adversarial jobs ran the full package with `COLDKEEP_TEST_DB=1` and +`COLDKEEP_LONG_RUN=1`. The plain package completed in 262.657 seconds and the +AES-GCM cell completed successfully as well. Because all three Phase 13A tests +are DB- and long-run-gated and have no additional skip path, the successful +full-package executions establish that the contention and both sequential +integrity tests executed rather than skipped. Failure-diagnostic collection +was not invoked in either cell. + +The outer codec matrix still runs the tests' inner plain/AES-GCM loop. Phase +13A intentionally preserves that duplicate coverage. + +## Stable Busy contract proved + +The real CLI contender against the holder's repository is required to produce: + +```text +process exit: 1 +stdout: empty +status: error +error_class: GENERAL +exit_code: 1 +error.code: REPOSITORY_BUSY +error.message: repository is busy +top-level message: repository is busy +logical-file rows before release: 0 +``` + +After `RELEASED`, the same real CLI store succeeds and its restored bytes match +the input SHA-256. + +## Proof and deferral boundary + +- **Linux independent-process repository exclusion:** PROVEN +- **Sequential real-CLI/PostgreSQL same-file convergence:** PROVEN +- **Sequential real-CLI/PostgreSQL shared-chunk integrity:** PROVEN +- **Killed-process release and reacquisition:** NOT PROVEN / PHASE 13B +- **Live-GC cross-process barrier:** NOT PROVEN / LATER PHASE 13 +- **PostgreSQL advisory lock on one dedicated `*sql.Conn`:** NOT PROVEN / LATER PHASE 13 +- **macOS/Windows independent-process contention:** NOT CLAIMED +- **Network/cross-host coordination:** NOT CLAIMED + +The helper's emergency kill path exists only to prevent a leaked test process; +it is not killed-holder semantic evidence. + +## Phase state + +- **Phase 12G:** COMPLETE +- **Phase 12H:** NOT STARTED +- **Phase 13A:** COMPLETE +- **Phase 13B:** NOT STARTED +- **BKC-016:** Deferred — documented diff --git a/docs/release/v1.13/v1.13.11-phase13b-killed-process-release.md b/docs/release/v1.13/v1.13.11-phase13b-killed-process-release.md new file mode 100644 index 00000000..076fc077 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase13b-killed-process-release.md @@ -0,0 +1,158 @@ +# Coldkeep v1.13.11 Phase 13B — Killed-Process Lease Release Evidence + +**Phase:** 13B — killed-process native Lease release and deterministic +reacquisition + +**Status:** Complete + +**Implementation commit:** `5bdaa44324d02b89ca97159cbdc22e2e46a459fe` + +**Hosted-lint correction:** `4aa2e75441d8f9f7f6624710b5afeccb9417be54` + +**Final implementation head:** `4aa2e75441d8f9f7f6624710b5afeccb9417be54` + +**Implementation scope:** Adversarial test/harness only + +## Implemented proof + +Phase 13B changed only +`tests/adversarial/g6_concurrent_operations_adversarial_test.go` in the two +implementation commits. It made no production Coordinator, native-lock, CLI, +owner-metadata, backend, workflow, or public API change. + +The implementation reuses the Phase 13A helper process and adds +`TestAdversarialG6KilledLeaseHolderReleasesRepository`. On Linux the test: + +1. starts the helper using the current adversarial test binary; +2. acquires a real production `Coordinator` Lease and waits for `READY`; +3. verifies `repository.lock` and the helper PID in `owner.json`; +4. requires a real CLI store to fail with the complete `REPOSITORY_BUSY` + contract and produce no logical-file row; +5. calls `Process.Kill` without sending `RELEASE`, then receives the existing + `Cmd.Wait` result and requires an unsuccessful `*exec.ExitError`; +6. verifies that `repository.lock` persists and the killed holder's valid but + stale `owner.json` remains; +7. immediately runs one real CLI store without sleep or retry; +8. requires successful JSON output and a valid logical-file ID; +9. verifies that normal successor release removes `owner.json` while leaving + `repository.lock`; and +10. runs the existing repository invariant and restore/SHA-256 checks. + +The holder clears `COLDKEEP_TEST_DB` and does not open the database or mutate +repository data. The parent fixture and real CLI commands retain the existing +isolated PostgreSQL G6 infrastructure. + +## Native release and metadata result + +The test proves this Linux transition: + +```text +production Lease held +→ real CLI REPOSITORY_BUSY +→ Process.Kill / SIGKILL +→ Cmd.Wait reaps holder +→ stale owner.json remains diagnostic only +→ first post-Wait real CLI acquisition succeeds +→ repository verifies and restores correctly +``` + +`repository.lock` remains a regular persistent artifact before death, after +death, and after successor release. The new acquisition succeeds while the +killed holder's metadata is still present. Production acquisition replaces +that record through the existing atomic publication path, and normal successor +release removes it. + +No PID-liveness, `/proc`, mtime, age, TTL, retry, sleep, force, break-lock, or +stale-owner cleanup behavior was added. Native kernel lock state remains the +only ownership authority. + +The Unix lock descriptor is opened with `O_CLOEXEC`, and the helper spawns no +descendants after acquisition. No surviving process retains the killed +holder's open file description. + +## Local validation + +The following validation passed at the final implementation state: + +- focused Linux/PostgreSQL proof, plain codec: `PASS` in `4.520s`; +- twenty consecutive focused repetitions: `PASS` in `59.522s`; +- focused race run: `PASS` in `2.969s`; +- post-lint-correction focused race run: `PASS` in `3.798s`; +- complete plain adversarial race suite: `PASS` in `539.294s`; +- complete AES-GCM adversarial race suite: `PASS` in `465.848s`; +- Darwin and Windows adversarial test-binary compile-only checks: `PASS`; +- `git diff --check`; +- `go mod verify`; +- `go mod tidy -diff`; +- `golangci-lint run` with zero issues; +- `go vet ./...`; +- `go test -count=1 ./...`; and +- `go test -race -count=1 ./...`. + +Every focused and repeated proof iteration used the exact sequence +`READY → Busy → Kill → Wait → immediate success`. No retry or arbitrary sleep +was used as a correctness mechanism. + +## Hosted validation + +Final-head CI run `31308921622` completed successfully: + +| Evidence | Result | +| --- | --- | +| `adversarial (plain)` job `93234838004` | Success | +| `adversarial (aes-gcm)` job `93234838028` | Success | +| Linux native job `93233617799` | Success | +| macOS native job `93233617752` | Success | +| Windows native job `93233617794` | Success | +| Required Gate job `93235437532` | Success | + +Both adversarial jobs ran the complete package with `COLDKEEP_TEST_DB=1` and +`COLDKEEP_LONG_RUN=1` on Linux. The killed-holder test has no further skip +condition after those gates and its Linux platform guard, so both successful +package executions establish that it executed rather than skipped. The plain +and AES-GCM G1–G17 steps passed, and failure-diagnostic collection was not +invoked. + +Final-head CodeQL run `31308921567` also completed successfully: + +| Evidence | Result | +| --- | --- | +| Actions analysis job `93233580887` | Success | +| Go analysis job `93233587128` | Success | +| Python analysis job `93233580873` | Success | +| CodeQL Aggregate job `93233693792` | Success | + +The first implementation-head CI run `31308831499` found Staticcheck +`QF1008` in the new test helper before the required adversarial jobs ran. +Commit `4aa2e75441d8f9f7f6624710b5afeccb9417be54` made the selector-only +correction, after which local lint and the complete final-head hosted matrix +passed. No semantic test or production behavior changed in that correction. + +## Proof boundary + +- **Linux independent-process fail-fast contention:** PROVEN / PHASE 13A +- **Linux killed-holder release and immediate reacquisition:** PROVEN / PHASE + 13B +- **Persistent lock artifact is not ownership:** PROVEN ON LINUX +- **Stale owner metadata is non-authoritative:** PROVEN ON LINUX +- **macOS killed-holder release:** NOT PROVEN / DEFERRED +- **Windows killed-holder release:** NOT PROVEN / DEFERRED +- **Live-GC cross-process barrier:** NOT PROVEN / LATER PHASE 13 +- **PostgreSQL advisory lock on one dedicated `*sql.Conn`:** NOT CLOSED / + LATER PHASE 13 +- **Cross-host/distributed coordination:** NOT CLAIMED +- **Network-filesystem safety:** NOT GUARANTEED +- **BKC-016:** Deferred — documented + +Phase 12 remains closed. Phase 13B did not reopen or modify its production +coordination contract. + +## Phase state + +- **Phase 12A–12H:** COMPLETE +- **Phase 13A:** COMPLETE +- **Phase 13B:** COMPLETE +- **Later Phase 13:** NOT STARTED +- **BKC-016:** Deferred — documented + +**Phase 13B closure verdict:** READY / COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase13c-live-gc-coordination.md b/docs/release/v1.13/v1.13.11-phase13c-live-gc-coordination.md new file mode 100644 index 00000000..ef5fbc2d --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase13c-live-gc-coordination.md @@ -0,0 +1,226 @@ +# Coldkeep v1.13.11 Phase 13C — Live-GC Coordination Evidence + +**Phase:** 13C — live-GC cross-process repository barrier and PostgreSQL +dedicated-session advisory locking + +**Status:** Complete + +**Dedicated-session implementation commit:** +`da78614af92835b78ebdf0ac84d94fd1f828f41d` + +**Independent-process proof commit:** +`54b1a04cdd1680f4b173d6ff9cd0c93adeca676f` + +**Final implementation head:** +`54b1a04cdd1680f4b173d6ff9cd0c93adeca676f` + +**Phase 13B evidence synchronized in the same implementation push:** +`701843243487bc9566df213ccfa0c070d44fa7de` + +## Implemented correction + +Phase 13C retains the existing top-level exclusive repository Lease for both +dry-run and live GC. It changes no Coordinator, native-lock, CLI policy, +schema, storage format, public API, or advisory-lock key. + +PostgreSQL GC now reserves one `*sql.Conn`, acquires the session-level lock +with `pg_try_advisory_lock(847362)` on that connection, and keeps the +connection checked out throughout GC preflight, planning, sweep, and cleanup. +It scans `pg_advisory_unlock(847362)` on the same connection and returns the +session to the pool only when the result is `true`. + +The cleanup contract is: + +```text +GC operation +→ advisory unlock on the owning session +→ normal connection return or forced physical-session discard +→ GC runtime/database cleanup +→ outer repository Lease release +``` + +An unlock SQL error or unexpected `false` is returned as a cleanup error. If +the GC operation also failed, `errors.Join` preserves both failures. When lock +ownership is uncertain, `Conn.Raw` returns `driver.ErrBadConn`, preventing the +physical session from re-entering the pool. The expected discard signal is not +reported as an additional user-facing error. Cleanup uses a fresh bounded +operation context. + +PostgreSQL live or dry-run GC now fails fast when +`DB.Stats().MaxOpenConnections == 1`, because the pinned advisory session and +GC work require at least two pool connections. Unlimited pools and pools with +at least two connections retain their existing behavior. SQLite live GC +remains explicitly unsupported, while SQLite dry-run remains supported. + +## Dedicated-session contract results + +The required PostgreSQL maintenance contracts are: + +- `TestGCAdvisoryLockUsesDedicatedSessionAndReleases`; +- `TestRunGCReleasesAdvisoryLockAfterOperationFailure`; +- `TestRunGCAdvisoryCleanupFailureReturnsErrorAndDiscardsSession`; and +- `TestRunGCLiveRefusesSingleConnectionPool`. + +The existing `TestRunGCRefusesWhenAdvisoryLockAlreadyHeld` fixture now also +pins its holder connection. + +The dedicated-session test observes the lock holder through `pg_locks`, reads +the acquisition backend PID, queries `pg_backend_pid()` through the unlock +connection, and requires exact PID equality. It also proves that another +session cannot acquire key `847362` while GC is held and can acquire it +immediately after successful cleanup. + +The operation-failure test forces GC preflight to fail and immediately +reacquires the key from an independent session. The cleanup-failure test covers +both an unlock SQL error and an unexpected `false`: each failure is returned, +the owning backend PID disappears from `pg_stat_activity`, and another session +immediately acquires the key. No uncertain locked session returns to the pool. + +The required plain-codec PostgreSQL CI step runs `go test -race -count=1 +-json` and validates an explicit pass event for every test selector above. +Consequently, correctness job `93240210968` proves that all four tests executed +and passed rather than skipped. + +## Independent-process live-GC proof + +`TestAdversarialG6LiveGCExcludesIndependentStoreProcess` uses a deterministic +PostgreSQL relation barrier on Linux: + +```text +parent transaction locks physical_file in ACCESS EXCLUSIVE mode +→ real coldkeep gc --output json starts +→ pg_stat_activity observes GC waiting on the physical-file preflight query +→ pg_blocking_pids identifies the parent barrier session +→ real coldkeep store receives REPOSITORY_BUSY +→ parent rolls back the relation barrier +→ GC completes successfully +→ immediate successor store succeeds +``` + +The server-observed wait occurs after advisory acquisition and while the real +CLI holds its outer repository Lease. Polling is bounded and predicate-based; +there is no arbitrary synchronization sleep. + +While GC is blocked, the test verifies the GC child PID and repository identity +in `owner.json`, requires `repository.lock` to remain a regular file, and +requires the contender's complete stable Busy contract: + +```text +process exit: 1 +stdout: empty +status: error +error_class: GENERAL +exit_code: 1 +message: repository is busy +error.code: REPOSITORY_BUSY +error.message: repository is busy +matching logical-file rows: 0 +``` + +After barrier release, the GC child must exit zero with `status=ok`, +`command=gc`, `data.dry_run=false`, and zero affected containers. Normal +completion removes `owner.json` and leaves `repository.lock` in place. The +first subsequent Store succeeds without retry, the bounded repository +invariants pass, and both the retained anchor and successor content restore to +their expected SHA-256 digests. + +The hosted adversarial jobs run the complete package on Linux with +`COLDKEEP_TEST_DB=1` and `COLDKEEP_LONG_RUN=1`. The new test has no remaining +skip condition after those gates and the Linux platform guard, so successful +plain and AES-GCM package execution establishes execution rather than skip. + +## Local validation + +Validation at the final implementation state passed as follows: + +- focused five-test PostgreSQL advisory contract suite: `PASS` in `0.944s`; +- focused live-GC process proof, plain codec: `PASS` in `2.527s`; +- twenty consecutive live-GC process repetitions: `PASS` in `35.883s`; +- focused advisory contract race suite: `PASS` in `2.088s`; +- focused live-GC process race proof: `PASS` in `3.009s`; +- PostgreSQL race regression across maintenance, engine, database, catalog, + and container packages: `PASS`; +- complete plain adversarial race suite: `PASS` in `287.448s`; +- complete AES-GCM adversarial race suite: `PASS` in `286.956s`; +- `git diff --check`; +- `go mod verify`; +- `go mod tidy -diff`; +- `go test -count=1 ./...`; +- `go test -race -count=1 ./...`; +- `go vet ./...`; and +- `golangci-lint run ./...` with zero issues. + +Every focused repetition completed the exact relation-lock, server-observed +wait, Busy, rollback, GC-success sequence. No retry or arbitrary sleep was used +as a correctness mechanism. + +## Hosted validation + +[CI run `31311628561`](https://github.com/franchoy/coldkeep/actions/runs/31311628561) +completed successfully at final implementation head `54b1a04`. + +| Evidence | Result | +| --- | --- | +| Quality job `93240210924` | Success | +| Correctness matrix plain `93240210968` | Success; required PostgreSQL contracts executed | +| Correctness matrix AES-GCM `93240210958` | Success | +| Integration stress plain `93240663124` | Success | +| Integration stress AES-GCM `93240663131` | Success | +| Integration long-run plain `93241207952` | Success | +| Integration long-run AES-GCM `93241207956` | Success | +| Adversarial plain `93241436681` | Success; live-GC process proof executed | +| Adversarial AES-GCM `93241436683` | Success; live-GC process proof executed | +| Linux native `93240210957` | Success | +| macOS native `93240210965` | Success | +| Windows native `93240210952` | Success | +| Benchmark integrity, all four cells `93240712246`, `93240712255`, `93240712256`, `93240712259` | Success | +| Benchmark timing advisory, all four cells `93240712278`, `93240712283`, `93240712286`, `93240712288` | Success | +| CI Required Gate `93242095156` | Success | + +Both adversarial jobs passed `Run deterministic G6 PostgreSQL interleaving +regression`, `Run adversarial validation (G1–G17)`, and the Phase 7 +snapshot-retention adversarial step. Failure-diagnostic collection was not +invoked. + +[CodeQL run `31311628559`](https://github.com/franchoy/coldkeep/actions/runs/31311628559) +also completed successfully: + +| Evidence | Result | +| --- | --- | +| Actions analysis `93240217817` | Success | +| Go analysis `93240210816` | Success | +| Python analysis `93240217826` | Success | +| CodeQL Aggregate `93240336496` | Success | + +## Proof boundary + +- **Linux independent-process fail-fast contention:** PROVEN / PHASE 13A +- **Linux killed-holder release and immediate reacquisition:** PROVEN / PHASE + 13B +- **Linux live-GC cross-process repository barrier:** PROVEN / PHASE 13C +- **PostgreSQL advisory acquisition/unlock session identity:** CLOSED / PHASE + 13C +- **PostgreSQL advisory success and operation-failure release:** PROVEN +- **PostgreSQL uncertain cleanup session discard:** PROVEN FOR SQL ERROR AND + FALSE UNLOCK RESULT +- **PostgreSQL single-connection pool boundary:** ENFORCED +- **SQLite live GC:** UNSUPPORTED / EXPLICITLY REJECTED +- **macOS killed-holder and live-GC process proof:** NOT PROVEN / DEFERRED +- **Windows killed-holder and live-GC process proof:** NOT PROVEN / DEFERRED +- **Cross-host/distributed coordination:** NOT CLAIMED +- **Network-filesystem safety:** NOT GUARANTEED +- **BKC-016:** Deferred — documented + +Phase 12 remains closed. Phase 13C closes its two planned proof gaps, but Phase +13 still requires its final reconciliation before the overall phase is closed. + +## Phase state + +- **Phase 12A–12H:** COMPLETE +- **Phase 13A:** COMPLETE +- **Phase 13B:** COMPLETE +- **Phase 13C:** COMPLETE +- **Phase 13 final reconciliation:** NOT YET RECORDED +- **BKC-016:** Deferred — documented + +**Phase 13C closure verdict:** READY / COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase14-container-range-header-consistency.md b/docs/release/v1.13/v1.13.11-phase14-container-range-header-consistency.md new file mode 100644 index 00000000..623e7b28 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase14-container-range-header-consistency.md @@ -0,0 +1,175 @@ +# Coldkeep v1.13.11 Phase 14 — Container Range and Header Consistency Evidence + +**Phase:** 14 — Container Range and Header Consistency Hardening + +**Status:** Complete + +**Range/header implementation commit:** +`fc88a4e90d8ca1dfad9f10caa8afbd619fae3dd0` + +**Recovery compatibility corrections:** +`314e2ad10e8003e19204e9e52fc225cdff7ef080`, +`2cc39c1119f790022c226976a28fd7e3214ac2e2`, and +`a576b19d5c75e912af3df0442b0486b06809d247` + +**Final implementation head:** +`a576b19d5c75e912af3df0442b0486b06809d247` + +## Implemented contract + +Phase 14 adds one subtraction-based outer-container range validator and applies +it before allocation or filesystem I/O. `FileContainer.ReadAt` rejects +negative, out-of-bounds, overflowing, and platform-`int`-unrepresentable +ranges before `make` or `ReadAt`. `ReadPayloadAt` additionally rejects nil +containers and offsets below the fixed 64-byte header. Zero-length reads at a +valid position, the first payload at byte 64, exact-EOF reads, and exact-limit +appends remain valid. A post-open truncation still fails through the existing +short-read check. + +Header decoding now requires the persisted maximum to be greater than the +fixed header length, including rejection of unsigned-to-signed wrap. Opening a +container requires a structurally valid catalog maximum, exact header/catalog +maximum equality, and a physical size no greater than that maximum. Header +serialization rejects invalid maxima and returns `io.ErrShortWrite` for a +nil-error short write. The final physical append gate uses overflow-safe +subtraction and retains `ErrContainerFull`. + +Packed reads now load `container.max_size` with their block metadata and carry +that persisted value through the verification reader. They no longer substitute +the current process-global container maximum, so repositories created under a +different valid configuration remain readable. + +Hosted regression exposed recovery paths that historically replaced a real +container's catalog maximum with its damaged physical/current size. Those paths +now preserve the persisted capacity while synchronizing `current_size` for +quarantine. Synthetic orphan-quarantine rows retain their established recovery +contract: when `current_size == max_size` identifies a physical-size marker, +both markers resynchronize together; a distinct real-container capacity is +preserved. + +The implementation changes no header bytes, CRC window, supported version, +schema, migration, public API, CLI surface, coordination behavior, +decompression behavior, or JSON rendering. + +## Focused contract results + +The new and extended deterministic tests cover: + +- `TestValidateContainerRangeBoundaries`; +- `TestFileContainerReadAtRejectsInvalidRangeBeforeAllocation`; +- `TestFileContainerReadAtAllowsZeroLengthAndExactEOF`; +- `TestFileContainerReadAtFailsClosedWhenFileShrinksAfterOpen`; +- `TestReadPayloadAtRejectsInvalidRangeBeforeRead`; +- `TestReadPayloadAtAllowsFirstPayloadAndExactEOF`; +- `TestReadAndValidateContainerHeaderRejectsInvalidMaxSizeAcrossSupportedVersions`; +- `TestWriteNewContainerHeaderRejectsInvalidMaxSize`; +- `TestWriteNewContainerHeaderRejectsShortWrite`; +- `TestOpenExistingContainerRejectsHeaderCatalogMaxSizeMismatch`; +- `TestOpenExistingContainerRejectsPhysicalSizeBeyondDeclaredMaximum`; +- `TestOpenExistingContainerAcceptsSupportedMatchingHeaderMaxSize`; +- `TestFileContainerAppendRejectsOverflowAsContainerFull`; +- `TestStorageBlockReaderLoadBlockMetadataIncludesTransformAwareFields`; +- `TestStorageBlockReaderUsesCatalogContainerMaxSize`; and +- `TestStorageBlockReaderRejectsContainerHeaderCatalogMaxSizeMismatch`. + +The no-allocation/no-delegate fixtures prove invalid outer ranges are rejected +before memory allocation and physical reads. Named header offsets and the +version-specific CRC helper drive v0/v1 corruption fixtures. Matching v0 and v1 +headers remain accepted, and header serialization remains byte-compatible. + +## Local validation + +The final implementation state passed: + +- the focused container and storage selectors above without skips; +- `go test -count=1 ./internal/container ./internal/storage ./internal/verify`; +- `go test -race -count=1 ./internal/container ./internal/storage ./internal/verify`; +- complete repository `go test -count=1 ./...` and + `go test -race -count=1 ./...` runs; +- `go vet ./...`, `go mod verify`, `go mod tidy -diff`, and + `git diff --check`; +- Darwin, Windows, and FreeBSD compile checks for `internal/container` and + `internal/storage`; +- release-state validator tests, automatic release-state validation, validation + matrix audit, and local CI-enforcement audit; +- PostgreSQL race regressions for the recovery compatibility paths in plain and + AES-GCM configurations; +- the exact G2 orphan-size recovery adversarial regression under `-race`; and +- the planned G7/G14 PostgreSQL corruption regressions under `-race`. + +Local changed-lines lint reported zero issues. The unscoped local lint command +also reported 35 pre-existing `SA5011` findings outside the Phase 14 diff; the +required hosted `quality` job ran the repository's canonical full lint step and +passed. + +## Hosted validation + +[CI run `31319918720`](https://github.com/franchoy/coldkeep/actions/runs/31319918720) +completed successfully at final implementation head `a576b19` after retrying a +single infrastructure failure. The first `quality` attempt timed out while +golangci-lint fetched its remote JSON schema, before lint execution. The retry +passed schema verification and every quality step without a code change. + +| Evidence | Result | +| --- | --- | +| Quality `93263322156` | Success; lint, vet, plain/AES-GCM package tests, builds, and audits passed | +| Correctness plain `93263322451` | Success | +| Correctness AES-GCM `93263322757` | Success | +| Integration stress plain `93263322967` | Success | +| Integration stress AES-GCM `93263322816` | Success | +| Integration long-run plain `93263323326` | Success | +| Integration long-run AES-GCM `93263323111` | Success | +| Adversarial plain `93263323394` | Success; G1–G17 and G14–G17 passed | +| Adversarial AES-GCM `93263323535` | Success; G1–G17 and G14–G17 passed | +| Linux native `93263322557` | Success | +| macOS native `93263333185` | Success | +| Windows native `93263322589` | Success | +| Legacy compatibility `93263848982` | Success | +| Smoke plain/AES-GCM `93263849012`, `93263849010` | Success | +| Benchmark integrity `93263849019`, `93263849040`, `93263849042`, `93263849069` | Success | +| Benchmark timing advisory `93263849016`, `93263849017`, `93263849024`, `93263849030` | Success | +| CI Required Gate `93264606698` | Success | + +[CodeQL run `31319918730`](https://github.com/franchoy/coldkeep/actions/runs/31319918730) +also completed successfully: + +| Evidence | Result | +| --- | --- | +| Go analysis `93261110986` | Success | +| Python analysis `93261111021` | Success | +| Actions analysis `93261111050` | Success | +| CodeQL Aggregate `93261256605` | Success | + +Earlier implementation-head runs served as fail-closed regression discovery: +they identified the recovery maximum rewrite and stale recovery assertions. +Each finding received a bounded correction and focused local reproduction +before the next push. Final-head correctness, integration, long-run, and both +adversarial codec jobs are green. + +## Compatibility and proof boundary + +- **Outer range validation before allocation/read:** PROVEN +- **Negative, overflowing, and platform-oversized range rejection:** PROVEN +- **Payload/header separation:** ENFORCED +- **Header/catalog/physical maximum consistency:** ENFORCED +- **Persisted catalog maximum in packed reads:** ENFORCED +- **Legacy v0 and current v1 compatibility:** PROVEN +- **On-disk format or migration change:** NONE +- **Decompression output/resource bounds:** DEFERRED TO PHASE 15 +- **Exact JSON integer fidelity:** DEFERRED TO PHASE 16 +- **Cross-host/distributed coordination:** NOT CLAIMED +- **Network-filesystem safety:** NOT GUARANTEED +- **BKC-016:** Deferred — documented + +## Phase state + +- **Phases 0–14:** COMPLETE +- **Phase 15 — Bounded Decompression:** NEXT +- **Phase 16 and later:** NOT STARTED +- **Release gate:** NOT STARTED +- **BKC-016:** Deferred — documented + +The v1.13.11 release remains active and incomplete. Phase 14 does not authorize +a PR, merge, tag, publication, or branch deletion. + +**Phase 14 closure verdict:** COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase15-bounded-decompression.md b/docs/release/v1.13/v1.13.11-phase15-bounded-decompression.md new file mode 100644 index 00000000..39226f9d --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase15-bounded-decompression.md @@ -0,0 +1,157 @@ +# Coldkeep v1.13.11 Phase 15 — Bounded Decompression Evidence + +**Phase:** 15 — Bounded Decompression + +**Status:** Complete + +**Implementation commit and final implementation head:** +`4213f3c581fbe3a8c9773f29dc5663e95d6d08ef` + +## Implemented contract + +Phase 15 implements the frozen exact-size-plus-absolute-maximum model for both +production compression codecs. Every call validates a known expected size in +`[0, 4 MiB]` before decoder creation or destination allocation. A decoder may +not produce more than the expected size or the fixed maximum, and success +requires the final output length to equal the expected size exactly. + +The identity codec applies the common expectation and final-size checks while +preserving its direct, no-copy return path. The zstd codec retains its per-call +decoder and whole-buffer API, but supplies an exact-capacity destination and +enables both `WithDecodeAllCapLimit(true)` and +`WithDecoderMaxMemory(4 MiB)`. Limit failures use the existing compression-size +mismatch contract; malformed, truncated, checksum-invalid, and invalid frames +use the existing decompression-failed contract. No partial output is returned. + +The 4 MiB ceiling is derived from released writer behavior. The largest packed +payload target is 3 MiB. At the smallest registered production chunk minimum +of 32 KiB, that payload has at most 96 entries. The complete CKBL maximum is +therefore `3,145,728 + 20 + 96*24 = 3,148,052` bytes, strictly below the +`4,194,304`-byte reader ceiling. Zstd shipped after the same packed-writer +restriction; released identity and zstd writers therefore remain compatible. + +The shared verification pipeline preserves fail-closed ordering: bounded +container read, physical hash, decrypt, compressed hash/size, bounded +decompression, exact plaintext size, logical hash, CKBL decode, and chunk +layout validation. Packed Restore, system Verify, and Store semantic reuse all +inherit this implementation. Existing Restore temporary-file cleanup and +destination atomicity remain unchanged. + +The implementation changes no on-disk bytes, schema, migration, public API, +dependency, CLI surface, JSON rendering, compression bytes, encryption bytes, +hash definitions, Phase 14 container behavior, coordination, or GC behavior. + +## Focused contract results + +The new deterministic tests cover: + +- `TestNoneDecompressRequiresExactExpectedSize`; +- `TestZstdDecompressRejectsExpectedSizeOutsideAbsoluteBound`; +- `TestZstdDecompressRejectsOutputBeyondExpectedSize`; +- `TestZstdDecompressRejectsTruncatedInput`; +- `TestZstdDecompressBoundsConcatenatedFramesAcrossAggregateOutput`; +- `TestStorageBlockReaderRejectsZstdOutputBeyondExpectedSizeAfterAESGCMDecrypt`; +- `TestRestorePackedZstdBoundFailurePreservesDestinationAndCleansTemp`; +- `TestValidateReusableLogicalFileForStoreRejectsZstdBoundFailure`; +- `TestMaximumReleasedPackedWriterOutputFitsDecompressionLimit`; +- `TestVerifyStoredBlockRejectsZstdOutputBeyondExpectedSizeAtDecompressStage`; + and +- `TestVerifyBlockPayloadsRejectsZstdOutputBeyondExpectedSize`. + +Together with retained coverage, these tests prove negative, `MaxInt64`, +over-maximum, zero, short, long, malformed, truncated, concatenated-frame, +identity, zstd, plain, AES-GCM, Restore, Verify, semantic-reuse, and +compatibility behavior. Small helper-level limits prove pre-allocation ordering +without near-OOM fixtures. + +## Local validation + +The final implementation state passed: + +- focused compression, storage, and verify selectors without skips; +- `go test -count=1 ./internal/storage/compression ./internal/storage ./internal/verify`; +- `go test -race -count=1 ./internal/storage/compression ./internal/storage ./internal/verify`; +- complete repository `go test -count=1 ./...` and + `go test -race -count=1 ./...` runs; +- `go vet ./...`, `go mod verify`, `go mod tidy -diff`, and + `git diff --check`; +- changed-lines golangci-lint with zero issues; +- Darwin, Windows, and FreeBSD compile checks for `internal/storage` and + `internal/verify`; +- release-state validator tests, automatic release-state validation, validation + matrix audit, local CI-enforcement audit, versioned-row-writer audit, and + smart-quote audit; and +- all four `BenchmarkVerifyPerformanceSanity` profiles with no new timing + threshold or performance accommodation. + +## Hosted validation + +[CI run `31326930293`](https://github.com/franchoy/coldkeep/actions/runs/31326930293) +completed successfully at final implementation head `4213f3c`. + +| Evidence | Result | +| --- | --- | +| Quality `93278679761` | Success; lint, vet, plain/AES-GCM package tests, builds, and audits passed | +| Correctness plain/AES-GCM `93278679775`, `93278679759` | Success | +| Integration stress plain/AES-GCM `93279180886`, `93279180883` | Success | +| Integration long-run plain/AES-GCM `93279996725`, `93279996712` | Success | +| Adversarial plain/AES-GCM `93280255523`, `93280255511` | Success | +| Linux/macOS/Windows native `93278679773`, `93278679747`, `93278679767` | Success | +| Legacy compatibility `93279257988` | Success | +| Smoke plain/AES-GCM `93279258004`, `93279258000` | Success | +| Benchmark integrity `93279258072`, `93279258084`, `93279258094`, `93279258091` | Success | +| Benchmark timing advisory `93279258043`, `93279258085`, `93279258055`, `93279258077` | Success | +| CI Required Gate `93281016993` | Success | + +[CodeQL run `31326930268`](https://github.com/franchoy/coldkeep/actions/runs/31326930268) +also completed successfully: + +| Evidence | Result | +| --- | --- | +| Go analysis `93278679617` | Success | +| Python analysis `93278679586` | Success | +| Actions analysis `93278679644` | Success | +| CodeQL Aggregate `93278816815` | Success | + +## Compatibility and proof boundary + +- **Expectation validation before decoder/allocation:** PROVEN +- **Identity exact-size contract with no-copy success:** ENFORCED +- **Zstd aggregate output cap:** ENFORCED +- **Zstd decoder memory/window cap:** ENFORCED +- **Concatenated-frame aggregate bypass:** REJECTED +- **Restore cleanup and destination atomicity:** PROVEN +- **Verify `metadata_invalid` / `decompress` classification:** PROVEN +- **Store semantic-reuse bound failure:** PROVEN +- **Released packed-writer maximum below reader ceiling:** PROVEN +- **Legacy and current packed compatibility:** PROVEN +- **On-disk format, schema, migration, or dependency change:** NONE +- **Exact JSON integer fidelity:** DEFERRED TO PHASE 16 +- **Fail-closed SQL mutation audit:** DEFERRED TO PHASE 17 +- **Cross-host/distributed coordination:** NOT CLAIMED +- **Network-filesystem safety:** NOT GUARANTEED +- **BKC-016:** Deferred — documented + +## Read-only audit + +The final implementation audit confirms validation precedes decoder creation +and destination allocation; both codecs enforce one exact contract; zstd +application output and decoder resources have independent bounds; aggregate +concatenated frames cannot bypass the cap; Restore, system Verify, and semantic +reuse use the shared path; valid legacy/current repositories remain accepted; +and no Phase 14, Phase 16, Phase 17, schema, CLI, coordination, GC, public API, +dependency, or on-disk-format work entered the slice. Fixtures are small and +deterministic. No unresolved Phase 15 defect remains. + +## Phase state + +- **Phases 0–15:** COMPLETE +- **Phase 16 — JSON Integer Fidelity:** NEXT +- **Phase 17 and later:** NOT STARTED +- **Release gate:** NOT STARTED +- **BKC-016:** Deferred — documented + +The v1.13.11 release remains active and incomplete. Phase 15 does not authorize +a PR, merge, tag, publication, or branch deletion. + +**Phase 15 closure verdict:** COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase16-json-integer-fidelity.md b/docs/release/v1.13/v1.13.11-phase16-json-integer-fidelity.md new file mode 100644 index 00000000..d4527660 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase16-json-integer-fidelity.md @@ -0,0 +1,171 @@ +# Coldkeep v1.13.11 Phase 16 — JSON Integer Fidelity Evidence + +**Phase:** 16 — JSON Integer Fidelity + +**Status:** Complete + +**Baseline:** `15b7e266d0a5569f8142d3f5b9b76522c89144e7` + +**Implementation commit and final implementation head:** +`5dcf0044a63ac10829212d0000d21f8d8d7d1197` + +## Confirmed pre-fix seam + +The stable v1.7 stats, inspect, and simulate-GC JSON renderers convert their +typed result into `map[string]any` so envelope-owned fields can be removed. +Before Phase 16, `toObjectMap` used `json.Unmarshal`, which materialized every +generic number as `float64`. Re-encoding then rounded integer tokens outside +the exact binary64 range. In particular, typed `int64(9007199254740993)` was +emitted from the helper as `9007199254740992` even though the original typed +marshal was exact. + +The affected public success surfaces were `stats --output json`/`--json`, all +`inspect` entities in JSON mode, and `simulate gc` in JSON mode, including +their nested maps and arrays. Other command success renderers, error envelopes, +trace JSONL, owner metadata, I/O counters, benchmark input/output, schemas, and +storage formats do not use this decode/remarshal seam and remain unchanged. + +## Implemented contract + +`toObjectMap` now decodes its internally marshaled bytes with +`json.Decoder.UseNumber()`. `UseNumber` applies recursively, so top-level, +nested-object, map, and array tokens remain `json.Number` through the existing +field-removal and final-envelope encoding paths. Integers therefore retain +their original exact decimal lexemes, while legitimate floats remain JSON +numbers and require no integer or floating-point parsing. + +After the first object decode, the helper attempts a second decode and requires +`io.EOF`. This preserves `json.Unmarshal`'s single-value plus trailing +whitespace behavior and rejects a trailing JSON value. Nil-map normalization, +error propagation, v1.7 envelope construction, encoder ordering/escaping, and +the terminating newline remain on their existing paths. + +Large integers remain JSON numbers, not strings. Coldkeep guarantees the exact +integer token it emits; it does not claim that downstream JavaScript clients +using ordinary `JSON.parse` retain integers outside JavaScript's safe range. + +## Deterministic test matrix + +`TestToObjectMapPreservesExactJSONNumbers` exercises the complete typed marshal, +generic decode, mutation-compatible object, and remarshal seam. Its integer +matrix is: + +- `0`, `1`, and `-1`; +- `2147483647` and `4294967296`; +- `9007199254740991`, `9007199254740992`, and `9007199254740993`; +- `-9007199254740993`; and +- `9223372036854775807` (`MaxInt64`). + +It also proves nested objects, an array of objects, ordinary integer `42`, +floats `0.5` and `1.25`, null, booleans, and strings. Assertions inspect +`json.Number.String()` and exact remarshal tokens; no assertion passes through +a generic `float64` conversion. + +`TestRunStatsCommandJSONPreservesExactLargeIntegers` uses the existing +`runObservabilityStatsPhase` injection seam and the real stats JSON command +renderer. It proves exact `2^53-1`, `2^53+1`, and `MaxInt64` tokens at stdout, +including a nested chunker-version element, without requiring a large database +or filesystem fixture. Existing stats, inspect, and simulate-GC schema, +determinism, category, single-object, and structured-error tests remain the +regression proof for all three renderers. + +## Local validation + +The final implementation state passed: + +- both planned focused renderer and command selectors; +- `go test -count=1 ./internal/cli/render ./cmd/coldkeep`; +- `go test -race -count=1 ./internal/cli/render ./cmd/coldkeep`; +- complete repository `go test -count=1 ./...` and + `go test -race -count=1 ./...` runs; +- `go vet ./...`, `go mod verify`, `go mod tidy -diff`, and + `git diff --check`; +- changed-lines golangci-lint with zero issues; +- Darwin, Windows, and FreeBSD compile checks for `internal/cli/render` and + `cmd/coldkeep`; and +- release-state validator tests, automatic release-state validation, validation + matrix audit, local CI-enforcement audit, versioned-row-writer audit, and + smart-quote audit. + +Go build and test cache/temp files were redirected to `/tmp`. PostgreSQL and +physical multi-petabyte fixtures were not required because the defect and fix +are confined to deterministic serialization of typed results. + +## Hosted validation + +[CI run `31332028771`](https://github.com/franchoy/coldkeep/actions/runs/31332028771) +completed successfully at implementation head `5dcf004`. + +| Evidence | Result | +| --- | --- | +| Quality `93291692262` | Success; lint, vet, plain/AES-GCM package tests, builds, and audits passed | +| Correctness plain/AES-GCM `93291692237`, `93291692223` | Success | +| Integration stress plain/AES-GCM `93292180172`, `93292180171` | Success | +| Integration long-run plain/AES-GCM `93292781217`, `93292781211` | Success | +| Adversarial plain/AES-GCM `93293045713`, `93293045721` | Success | +| Linux/macOS/Windows native `93291692311`, `93291692321`, `93291692327` | Success | +| Legacy compatibility `93292281737` | Success | +| Smoke plain/AES-GCM `93292281916`, `93292281920` | Success | +| Critical coverage `93292281886` | Success | +| Benchmark integrity `93292281754`, `93292281906`, `93292281753`, `93292281750` | Success | +| Benchmark timing advisory `93292281888`, `93292281740`, `93292281752`, `93292281910` | Success | +| CI Required Gate `93293724282` | Success | + +[CodeQL run `31332028770`](https://github.com/franchoy/coldkeep/actions/runs/31332028770) +also completed successfully: + +| Evidence | Result | +| --- | --- | +| Go analysis `93291692169` | Success | +| Python analysis `93291692125` | Success | +| Actions analysis `93291692143` | Success | +| CodeQL Aggregate `93291816510` | Success | + +## Compatibility and excluded boundaries + +- **Exact emitted integer tokens:** ENFORCED +- **Integer JSON type:** NUMBER, UNCHANGED +- **Nested/map/array fidelity:** PROVEN +- **Ordinary float JSON type and lexeme:** PRESERVED +- **Null/string/boolean/object behavior:** PRESERVED +- **Envelope fields, nesting, and `v1.7`:** UNCHANGED +- **Escaping, deterministic ordering, and trailing newline:** UNCHANGED +- **Error codes, messages, exit codes, and stderr routing:** UNCHANGED +- **Stats/inspect coordinated spool capture and replay:** UNCHANGED +- **Database schema or migration:** NONE +- **On-disk or storage format:** NONE +- **Public Go API or command result structure:** NONE +- **Dependency or workflow change:** NONE +- **Coordination, GC execution, or decompression change:** NONE +- **Phase 17 SQL mutation behavior:** NOT STARTED +- **Downstream JavaScript integer semantics:** NOT CLAIMED +- **BKC-016:** Deferred — documented + +## Read-only audit + +The final implementation audit confirms no `float64` intermediate remains in +the affected generic renderer path; positive and negative values beyond the +binary64 safe range and `MaxInt64` remain exact; recursion preserves nested and +array numbers; small integers and legitimate floats remain numeric; and +null/string/boolean/object behavior is stable. Existing renderer tests confirm +the envelope schema/version, ordering, escaping, and newline behavior. Existing +command tests confirm structured errors and stdout/stderr separation. + +Only the central helper and its focused renderer/command tests changed in the +implementation commit. Coordination and the output spool, schemas, migrations, +storage formats, public APIs, dependencies, workflows, decompression, and Phase +17 SQL mutation behavior did not enter the slice. No unresolved Phase 16 +finding remains. + +## Phase state + +- **Phases 0–16:** COMPLETE +- **Phase 17 — Fail-Closed SQL Mutation Audit:** NEXT +- **Phases 18–20:** NOT STARTED +- **Release gate:** NOT STARTED +- **BKC-016:** Deferred — documented + +The v1.13.11 release remains active and incomplete. Phase 16 does not authorize +a PR, merge, tag, publication, or branch deletion. + +**Phase 16 closure verdict:** COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase17-fail-closed-sql-mutations.md b/docs/release/v1.13/v1.13.11-phase17-fail-closed-sql-mutations.md new file mode 100644 index 00000000..5f294606 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase17-fail-closed-sql-mutations.md @@ -0,0 +1,235 @@ +# Coldkeep v1.13.11 Phase 17 — Fail-Closed SQL Mutation Audit Evidence + +**Phase:** 17 — Fail-Closed SQL Mutation Audit + +**Status:** Complete + +**Baseline:** `eff7ae5541865e9367bdbffc352887a70b521ed2` + +**Implementation commit and final implementation head:** +`0c50113c2ea6cbcd2c612f4d88fca7a9d1ab3336` + +## Implemented contract + +Phase 17 adds the internal `db.ErrMutationCardinality` sentinel and the +`RequireRowsAffected` / `RequireExactlyOneRow` validators. The validator calls +`RowsAffected` exactly once, fails closed when the driver cannot report a +count, and wraps the sentinel when the direct target count differs. Its error +text contains only a bounded logical operation label plus expected and actual +counts; it contains no SQL text, connection data, path, hash, or bound value. + +The 20 required mutations invoke the validator immediately after successful +execution and before later mutation or commit. Exact-N repair uses the +precomputed mismatch count. Existing RETURNING, upsert, CAS, bulk, cleanup, +recovery, and GC predicates remain unchanged. No retry or post-mutation count +query was added. + +The physical-file metadata UPDATE required one portability correction in +addition to its cardinality check. Its original PostgreSQL-style placeholders +appeared as `$2` through `$7` before `$1`. PostgreSQL resolves those numeric +indices, but SQLite treats `$name` parameters in first-occurrence order. The +statement could therefore bind the path into the SET list and bind the final +boolean into the path predicate, silently affecting zero rows. Phase 17 +renumbers the same SET columns and path predicate into occurrence order and +reorders the existing arguments. The predicate, target columns, transaction, +and public behavior are otherwise unchanged. The dual-backend same-value +contract and physical repository regressions cover this boundary. + +## Complete production mutation inventory + +The final read-only audit re-enumerated the same 70 non-DDL production +mutation sites. No mutation statement was added or removed. + +| Package | INSERT | UPDATE | DELETE | Total | +| --- | ---: | ---: | ---: | ---: | +| `internal/blocks` | 1 | 0 | 0 | 1 | +| `internal/container` | 2 | 5 | 0 | 7 | +| `internal/maintenance` | 0 | 2 | 6 | 8 | +| `internal/recovery` | 1 | 8 | 0 | 9 | +| `internal/snapshot` | 5 | 0 | 2 | 7 | +| `internal/storage` | 10 | 20 | 8 | 38 | +| **Total** | **19** | **35** | **16** | **70** | + +- 64 sites support SQLite and PostgreSQL; six live-GC sites are PostgreSQL + production paths. +- 13 sites are defined upserts and ten use RETURNING. +- 20 sites require Phase 17 enforcement, 18 are intentionally zero-row-safe, + and 32 were already enforced, RETURNING-proven, constraint-proven, + upsert-defined, or structurally safe. +- DDL, bootstrap, migration, and test SQL remains excluded. Catalog remains + read-only. + +### Required mutations — hardened + +| ID | Logical mutation | Required count | Disposition | +| --- | --- | ---: | --- | +| M17-001 | container size update | 1 | exact-one validator | +| M17-002 | container seal publication | 1 | exact-one validator | +| M17-003 | local-writer rotation sealing marker | 1 | exact-one validator | +| M17-004 | simulated-writer seal publication | 1 | exact-one validator | +| M17-005 | quarantined orphan resynchronization | 1 | exact-one validator | +| M17-006 | logical refcount repair | precomputed N | exact-N validator | +| M17-007 | chunk live-refcount repair | precomputed N | exact-N validator | +| M17-008 | sealed GC container delete | 1 | exact-one validator | +| M17-009 | selected packed storage-block delete | 1 | exact-one validator | +| M17-010 | fully dead active GC container delete | 1 | exact-one validator | +| M17-011 | physical-file metadata update | 1 | placeholder parity fix plus exact-one validator | +| M17-012 | logical-file refcount increment | 1 | exact-one validator | +| M17-013 | conditional logical-file refcount decrement | 1 | exact-one validator | +| M17-014 | physical-file replacement delete | 1 | exact-one validator | +| M17-015 | final remove-by-ID logical-file delete | 1 | exact-one validator | +| M17-016 | packed chunk completion | 1 | exact-one validator | +| M17-017 | reclaimed chunk completion | 1 | exact-one validator | +| M17-018 | positive-ID container sealing marker | 1 | exact-one validator | +| M17-019 | newly linked chunk live-ref increment | 1 | exact-one validator | +| M17-020 | final logical-file completion | 1 | exact-one validator | + +### Intentionally zero-row-safe mutations — unchanged + +- Container quarantine after an absent initial lookup. +- Snapshot membership bulk deletion for an empty snapshot. +- Recovery stale-marker sweep; already-absent quarantine targets; and the + sealing, missing-file, corrupt-tail, and orphan reconciliation no-op paths. +- GC conditional chunk deletion and bulk legacy-block / packed-reference + cleanup. +- Empty exact-N repair where the precomputed count is zero. +- Store failed-flush and deferred-abort cleanup, cleanup refcount decrement, + stale file-chunk cleanup, and legacy/packed rebuild metadata cleanup. +- CAS losers that reconcile current state and every documented + `ON CONFLICT DO NOTHING` branch. + +### Already enforced or structurally safe mutations — unchanged + +- Block insert-upsert and container creation require a RETURNING row. +- Snapshot path upserts verify resolved identities; snapshot and membership + inserts require success or RETURNING; snapshot deletion already requires one + snapshot row. +- Recovery bulk abort counts are consumed as statistics and orphan insertion + consumes its 0/1 conflict result. +- Repository configuration and packed-storage companion upserts have defined + conflict semantics. +- Physical-file insertion consumes its conflict count. Remove-by-path, + remove refcount decrements, restore pin/unpin, rebuild status transitions, + logical/chunk claims and retries, and file-chunk identity conflicts already + validate or reconcile cardinality. +- Store retry-marker ownership is file-chunk-FK-proven; packed storage-block + insertion requires RETURNING. + +These three groups account for all 70 sites: 20 hardened + 18 zero-row-safe + +32 previously enforced or structurally safe. + +## Regression matrix + +| Test | Proof | +| --- | --- | +| `TestMutationRowsAffectedContractAcrossBackends` | SQLite/PostgreSQL existing, missing, same-value, delete, upsert-conflict, nil-result, and unsupported-result behavior | +| `TestRequiredContainerMutationsFailClosedOnMissingRows` | missing size and simulated-seal targets | +| `TestSealContainerFailsClosedWhenUpdateMatchesZero` | ignored seal UPDATE returns the sentinel and rolls back | +| `TestLocalWriterRotationFailsClosedWhenSealingMarkerMatchesZero` | rotation fails before physical finalization or acknowledgment | +| `TestPhysicalFileMutationsFailClosedOnCardinalityMismatch` | update, increment, decrement, replacement delete, rollback, and SQLite placeholder parity | +| `TestOrphanResyncFailsClosedWhenUpdateMatchesZero` | ignored orphan resync cannot report success | +| `TestRepairRefCountsFailsClosedOnAffectedCountMismatch` | logical/chunk exact-N mismatch and rollback | +| `TestGCRequiredDeletesFailClosedOnAffectedCountMismatch` | sealed container, packed block, and active container rollback; physical files remain | +| `TestStoreRequiredMutationCardinalityFailuresRollBack` | sealing, linked refcount, chunk publication, and empty missing logical completion | +| `TestRemoveFileFailsClosedWhenLogicalDeleteMatchesZero` | final logical delete mismatch and rollback | + +Fixtures use current-schema temporary SQLite databases. `BEFORE UPDATE` and +`BEFORE DELETE` triggers with `RAISE(IGNORE)` produce successful SQL execution +with zero affected rows. A synthetic `sql.Result` is confined to the helper's +unsupported-result test. + +## Local validation + +The final implementation head passed: + +- the exact ten-test focused selector across `internal/db`, + `internal/container`, `internal/storage`, `internal/recovery`, and + `internal/maintenance`; +- all SQLite affected packages, plus `internal/snapshot` and + `internal/blocks`; +- the focused PostgreSQL command with the configured local test container; +- complete repository `go test -count=1 ./...` and + `go test -race -count=1 ./...` runs; +- affected-package race tests; +- `go vet ./...`, `go mod verify`, `go mod tidy -diff`, and + `git diff --check`; +- changed-lines golangci-lint with zero issues; +- Darwin, Windows, and FreeBSD compile checks for all five affected packages; + and +- release-state validator tests, automatic release-state validation, + validation-matrix audit, local CI-enforcement audit, + versioned-row-writer audit, and smart-quote audit. + +The locally installed broad golangci-lint invocation also reported 35 +pre-existing `SA5011` findings in untouched test files. No Phase 17 file was +reported, changed-lines lint was clean, and the hosted pinned quality lint job +passed the complete implementation head. No unrelated lint cleanup entered +this phase. + +## Hosted validation + +[CI run `31867193430`](https://github.com/franchoy/coldkeep/actions/runs/31867193430) +completed successfully at implementation head `0c50113`. + +| Evidence | Result | +| --- | --- | +| Quality `94970045145` | Success; lint, vet, plain/AES-GCM tests, builds, and audits passed | +| Correctness plain/AES-GCM `94970045129`, `94970045107` | Success; plain included PostgreSQL internal package contracts | +| Cross-platform Linux/macOS/Windows `94970045123`, `94970045154`, `94970045190` | Success | +| Integration stress plain/AES-GCM `94970535942`, `94970535908` | Success | +| Integration long-run plain/AES-GCM `94971109952`, `94971109928` | Success | +| Adversarial plain/AES-GCM `94971350435`, `94971350487` | Success | +| Smoke plain/AES-GCM `94970551333`, `94970551336` | Success | +| Critical coverage `94970551362` and legacy compatibility `94970551366` | Success | +| Benchmark integrity `94970551383`, `94970551437`, `94970551405`, `94970551388` | Success | +| Benchmark timing advisory `94970551344`, `94970551355`, `94970551348`, `94970551343` | Success | +| CI Required Gate `94972031158` | Success | + +[CodeQL run `31867193369`](https://github.com/franchoy/coldkeep/actions/runs/31867193369) +also completed successfully: + +| Evidence | Result | +| --- | --- | +| Actions analysis `94970044976` | Success | +| Python analysis `94970045009` | Success | +| Go analysis `94970045034` | Success | +| CodeQL Aggregate `94970174561` | Success | + +## Read-only final audit + +- All 70 production mutations remain inventoried and classified. +- Every required mutation is validator-, RETURNING-, existing-count-, or + structural-FK-proven. +- The 13 remaining direct production `RowsAffected` calls all check the + returned error; the shared validator supplies the fourteenth production + call and never discards its error. +- Every new mismatch check runs before commit. GC mismatch exits before + physical-file removal; Store mismatch uses existing rollback and writer + rollback/quarantine handling. +- Zero-safe cleanup, recovery, CAS, upsert, bulk, and GC behavior did not + change. +- SQLite and PostgreSQL report existing/missing direct targets as 1/0; + same-value updates report the matched row; upsert conflict reports zero; and + RETURNING remains row-or-`sql.ErrNoRows`. +- No schema, migration, schema version, dependency, public API, CLI code, + retry policy, workflow, coordination, Phase 12–16, or Phase 18 + implementation changed. + +## Compatibility and phase state + +- **Database schema or migration:** NONE +- **Dependency:** NONE +- **Public API, CLI schema, code, or exit behavior:** UNCHANGED +- **Retry or transaction topology:** UNCHANGED +- **Coordination and live-GC advisory ownership:** UNCHANGED +- **Container format, decompression, and JSON rendering:** UNCHANGED +- **Phases 0–17:** COMPLETE +- **Phase 18 — Required Backend and Coordination CI Gate:** NEXT +- **Phases 19–20:** NOT STARTED +- **Release gate:** NOT STARTED +- **BKC-016:** Deferred — documented + +The v1.13.11 release remains active and incomplete. Phase 17 does not +authorize a PR, merge, tag, publication, or branch deletion. + +**Phase 17 closure verdict:** COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase18-required-backend-coordination-ci.md b/docs/release/v1.13/v1.13.11-phase18-required-backend-coordination-ci.md new file mode 100644 index 00000000..622cd06f --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase18-required-backend-coordination-ci.md @@ -0,0 +1,120 @@ +# Coldkeep v1.13.11 Phase 18 — Required Backend and Coordination CI Gate + +**Phase:** 18 — Required Backend and Coordination CI Gate + +**Status:** Complete + +**Baseline:** `de741d58f31456763302201ce87757297c8d716f` + +**Implementation commits:** +`975453d76ac46389316957384e4bacb079e0e2ce` (`ci: require backend and coordination contracts`) +and `eaa5896fe0e61998c45683db29e0e473cff42609` (`ci: preserve audit status patterns`) + +## Implemented CI proof + +Phase 18 makes selected existing backend and coordination execution fail closed +in the existing required CI jobs. It adds no job, matrix, runner, service, +artifact, production, schema, migration, dependency, or public-API change. + +- The plain PostgreSQL internal-contract parser requires + `TestMutationRowsAffectedContractAcrossBackends/postgres` to pass. +- The correctness JSON parser requires `TestRoundTripStoreRestore` in both + codecs, and `TestRemoveWithSharedChunksRefCount` plus + `TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState` in the + plain codec. Missing, malformed, or skipped events fail the step. +- The existing adversarial full suite now emits JSON and requires passing G6 + independent-process plain/AES-GCM subtests, killed-holder release, and + live-GC exclusion. The proof is bound to the Ubuntu PostgreSQL step with + `COLDKEEP_TEST_DB=1` and `COLDKEEP_LONG_RUN=1`; matching skips fail it. +- The CI audit retains the two broad SQLite quality commands, checks the new + PostgreSQL and coordination selectors and gates, and retains the native Unix, + Windows, and production Coordinator test names. Its table-driven regression + test rejects removal or mutation of each Phase 18 invariant. + +The follow-up commit is a ShellCheck-only audit-script correction: two scoped +`SC2016` suppressions preserve literal status-pattern regexes. It does not +change the workflow topology or any execution-proof contract. + +## Hosted implementation-head evidence + +[CI run `31872189672`](https://github.com/franchoy/coldkeep/actions/runs/31872189672) +completed successfully at final implementation head `eaa5896`. + +| Evidence | Result | +| --- | --- | +| Quality `94982497983` | Success; broad SQLite plain and AES-GCM package paths remained required | +| Correctness AES-GCM `94982497926`; plain `94982497945` | Success; required JSON proof events passed | +| Cross-platform Ubuntu `94982497938`; Windows `94982497946`; macOS `94982497956` | Success; native runtime and production Coordinator tests passed | +| Adversarial AES-GCM `94983828625`; plain `94983828682` | Success; required Linux PostgreSQL G6 events passed in both cells | +| Integration stress `94983011892`, `94983011942`; long-run `94983579655`, `94983579661`; smoke `94983018598`, `94983018624`; legacy `94983018586` | Success | +| Benchmark integrity `94983018612`, `94983018637`, `94983018665`, `94983018695`; timing advisory `94983018611`, `94983018638`, `94983018644`, `94983018670` | Success | +| CI Required Gate `94984536507` | Success | + +The hosted JSON logs record these required `pass` events with no matching skip: + +- plain internal PostgreSQL contract: + `TestMutationRowsAffectedContractAcrossBackends/postgres`; +- both correctness cells: `TestRoundTripStoreRestore`; plain additionally: + `TestRemoveWithSharedChunksRefCount` and + `TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState`; +- both adversarial cells: + `TestAdversarialG6IndependentProcessRepositoryContention/plain`, + `TestAdversarialG6IndependentProcessRepositoryContention/aes-gcm`, + `TestAdversarialG6KilledLeaseHolderReleasesRepository`, and + `TestAdversarialG6LiveGCExcludesIndependentStoreProcess`; +- plain PostgreSQL internal contracts also retained the four advisory-session + events: `TestGCAdvisoryLockUsesDedicatedSessionAndReleases`, + `TestRunGCReleasesAdvisoryLockAfterOperationFailure`, + `TestRunGCAdvisoryCleanupFailureReturnsErrorAndDiscardsSession`, and + `TestRunGCLiveRefusesSingleConnectionPool`. + +[CodeQL run `31872189661`](https://github.com/franchoy/coldkeep/actions/runs/31872189661) +also succeeded; Actions `94982462204`, Python `94982462117`, Go `94982462215`, +and CodeQL Aggregate `94982589486` were successful. CodeQL remains separate +from `ci-required`. + +## Governance and boundaries + +`ci-required.needs` and its exact-success result policy are unchanged. +Benchmark integrity remains hard-required; hosted timing remains advisory, +while malformed evidence or evaluator failure remains blocking. Hard timing +regression enforcement remains deferred to controlled infrastructure, and +BKC-016 remains `Deferred — documented`. + +This is selected SQLite/PostgreSQL execution proof, not a broad backend-parity +claim. Quality remains the broad SQLite path; PostgreSQL storage and recovery +are proven through their existing named integration paths. Native runtime and +production Coordinator evidence remains required on Ubuntu, macOS, and +Windows. macOS/Windows subprocess duplication, cross-host coordination, and +network-filesystem guarantees remain outside the v1.13.11 contract. + +## Local validation and audit + +The final implementation boundary passed module verification and tidy diff, +release-state tests and validation, validation-matrix audit, shell syntax, +local CI-enforcement audit, the audit regression suite, native coordination, +and focused PostgreSQL internal, correctness, storage/recovery, adversarial, +and advisory-session commands. The read-only audit confirmed that only the +three authorized workflow/audit paths changed and that required-gate topology, +benchmarks, CodeQL separation, actions, permissions, caches, artifacts, and +Phase 19/20 scope remained unchanged. + +One initial local full-correctness attempt encountered a Docker-created, +unwritable default `storage/containers` directory and reported repository +identity invalid. Re-running with an isolated writable `/tmp` storage path +passed. This is a **LOCAL ENVIRONMENT / FILE OWNERSHIP ARTIFACT**, +**NON-PRODUCT**, and **NON-BLOCKING**; no code or CI change was made. + +## Compatibility and phase state + +- **Production code, product tests, schema, migrations, dependencies, public API, and storage format:** UNCHANGED +- **Phases 0–18:** COMPLETE +- **Phase 19 — Validation Matrix and Release Evidence Reconciliation:** NEXT +- **Phase 20:** NOT STARTED +- **Release gate:** NOT STARTED +- **BKC-016:** Deferred — documented + +The v1.13.11 release remains active and incomplete. Phase 18 does not +authorize a PR, merge, tag, publication, or branch deletion. + +**Phase 18 closure verdict:** COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase19-validation-evidence-reconciliation.md b/docs/release/v1.13/v1.13.11-phase19-validation-evidence-reconciliation.md new file mode 100644 index 00000000..05268129 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase19-validation-evidence-reconciliation.md @@ -0,0 +1,186 @@ +# Coldkeep v1.13.11 Phase 19 — Validation and Evidence Reconciliation + +**Phase:** 19 — Validation Matrix and Release Evidence Reconciliation + +**Status:** Complete + +**Baseline:** `9c1fa524a2b82a1ce7534acf3308ef75d0d72ac6` + +**Closure authority:** documentation and aggregate evidence only + +## Reconciliation decision + +Phase 19 is **READY / COMPLETE**. Repository claims for v1.13.11 are +reconciled through Phase 18 and are sufficient to enter Phase 20. No unresolved +product or CI correctness defect was found. + +Phase 19 changes documentation only. It changes no production code, product +test, CI workflow or enforcement script, schema, migration, dependency, public +API, storage format, repository format, or runtime behavior. Historical Phase +0–18 evidence remains unchanged. + +The release remains active and incomplete. Phase 20 — Final Exact-Head Local +Release Gate and PR Authorization — is Next. The release gate remains Not +started; no PR, merge, tag, publication, or branch deletion is authorized by +this evidence. + +## Documents audited + +The audit covered `VALIDATION_MATRIX.md`, `PRE_RELEASE_CHECKLIST.md`, the +v1.13.11 scope, phase list, validation checklist, release gate, backend claim +matrix, Phase 0–18 evidence documents, the v1.13 README, and the authoritative +v1.13 release train. It also inspected the validation-matrix validator, +release-state validator and fixture suite, local CI-enforcement audit, current +workflow topology, representative implementation/tests, and all relative +Markdown links among the active release documents. + +## Reconciliation findings + +| ID | Severity | Finding | Resolution | +| --- | --- | --- | --- | +| R19-001 | Medium | The validation matrix used a single `covered` concept, overgeneralized all evidence as integration/repeated-lifecycle proof, and stopped its maintained history before v1.13.11 hardening. | Added explicit unit, integration, adversarial, cross-platform, required-CI, advisory, and deferred classifications plus the Phase 14–18 proof groups. | +| R19-002 | Medium | G6 and Open Work still described process contention as unproven/deferred. | Preserved the in-process guarantee while recording native three-platform runtime, Linux independent-process/killed-holder/live-GC proof, Phase 18 named events, and the unsupported boundaries. | +| R19-003 | Medium | Stored-path remove dry-run remained listed as deferred. | Marked it completed by the v1.13.8 `Engine.RemoveStoredPaths`/CLI/test evidence. | +| R19-004 | High | The reusable pre-release checklist treated historical timing thresholds as hard CI-equivalent gates. | Replaced that block with hard candidate integrity and valid timing-advisory evaluation for all four profiles; hard timing enforcement remains deferred. | +| R19-005 | Medium | The backend claim matrix stopped at Phase 11 and left BKC-014–016 process/advisory proof pending. | Updated its evidence basis through Phase 18 and reconciled the bounded live-GC, advisory-session, and same-host coordination proof. BKC-016 remains `Deferred — documented`. | +| R19-006 | Medium | The authoritative release train still pointed to Phase 1. | Advanced it to Phases 0–19 Complete and Phase 20 Next. | +| R19-007 | Medium | Current aggregate documents omitted Phase 18 closure-head CI/CodeQL. | Recorded closure commit `9c1fa524`, CI `31873436272`, Required Gate `94987546529`, CodeQL `31873436304`, and Aggregate `94985540614`. | +| R19-008 | Medium | The release gate grouped Codacy with required authority. | Retained required static analysis and CodeQL; classified Codacy as advisory signal if available. | + +No Critical finding, broken active evidence link, missing evidence file, or +real release blocker was found. + +## Evidence artifact inventory + +| Phase | Artifact purpose | Implementation/evidence heads | Canonical hosted evidence | Current authority | +| --- | --- | --- | --- | --- | +| 0 | v1.13.10 closure correction and v1.13.11 baseline | `c4a21620`; released baseline `423c5781` | None required | Historical baseline | +| 1 | Release identity activation | `40a9462a` | Local identity checks | Historical implementation evidence | +| 2 | Backend compatibility claim matrix | `f748675d`; aggregate updates through this phase | Evidence through Phase 18 | Active aggregate contract | +| 3 | Reusable dual-backend harness | `7259b9de` | None required | Historical implementation evidence | +| 4 | PostgreSQL package CI activation | `0147851d`; evidence `6e1a0675` | CI `29729981751` | Authoritative CI evidence | +| 5 | Schema/migration contracts and G6 remediation | `2b603b7c`, `bfe49176`, `54ecd84c`; closure `460ff671` | CI `29815330238`, attempt 2 | Final run supersedes only the earlier aggregate failure | +| 6 | Implemented catalog parity | `c03c42bd`, `db12c3d2`; evidence `50517525` | CI `29983479388` | Bounded parity evidence | +| 7 | Engine read parity | `3e8c98df`, `313d0069`; evidence `ab8accb8` | CI `29993172886` | Bounded parity evidence | +| 8 | Selector determinism | `bcae3576`; evidence `8a7b9980` | CI `30109561344` | Authoritative | +| 9 | Mutation parity | `848e579b`; evidence `e4c77607` | CI `30114444798` | Scoped parity evidence | +| 10 | Transaction and row-lock semantics | `ad82c959`; evidence `81dd0da6` | CI `30148670910` | Backend-specific proof | +| 11 | Coordination contract and final benchmark governance | contract `84d4ce7e`; accepted head `b08da99a`; closure `d871ad5c` | CI `31242784813`; Gate `93068202252`; CodeQL `31242784806` | Governance/closure authoritative; calibration and paired artifacts remain historical/intermediate | +| 12 | Native runtime, Coordinator, and Phase 12 closure | `6a368048`, reconciliation `cf755305`; evidence `eba01b22`; closure `c1b5e4d0` | Initial CI `31255838594`; successful reconciliation `31258313120` | Initial native proof retained; later run closes aggregate/G6 result | +| 13 | Independent-process, killed-holder, live-GC, and advisory-session closure | `cf755305`, `5bdaa443`, `4aa2e754`, `da78614a`, `54b1a04c`; evidence `d4b9caee`; closure `aa0a9df0` | CI `31258313120`, `31308921622`, `31311628561`, evidence-head `31312520061` | Aggregate closure authoritative; subphase artifacts retain chronology | +| 14 | Container range/header consistency | final implementation `a576b19d`; closure `a5ab52de` | CI `31319918720`; CodeQL `31319918730` | Authoritative | +| 15 | Bounded decompression | `4213f3c5`; closure `15b7e266` | CI `31326930293`; CodeQL `31326930268` | Authoritative | +| 16 | JSON integer fidelity | `5dcf0044`; closure `eff7ae55` | CI `31332028771`; CodeQL `31332028770` | Authoritative | +| 17 | Fail-closed SQL mutation audit | `0c50113c`; closure `de741d58` | CI `31867193430`; CodeQL `31867193369` | Authoritative within selected backend contracts | +| 18 | Required backend and coordination CI | `975453d7`, correction `eaa5896f`; closure `9c1fa524` | Implementation CI `31872189672`; closure CI `31873436272` | Historical Phase 18 artifact plus this aggregate closure chronology | + +Later evidence supersedes only the earlier pending or aggregate-result state. +It does not rewrite the facts, boundaries, or intermediate failures recorded by +the historical artifact. + +## Final proof matrix after Phases 12–18 + +| Validation group | Proof | Required hosted boundary | +| --- | --- | --- | +| G1–G5 storage/recovery | Unit, integration, and adversarial proof; selected Phase 18 storage/recovery events | Named selected events plus broad required jobs; single-node/local-filesystem semantics | +| G6 coordination | Same-process protection; native Linux/macOS/Windows runtime; production Coordinator lifecycle on all three; Linux independent-process contention, killed-holder release, and live-GC exclusion | Native three-platform jobs and named Linux pass events; no separate macOS/Windows subprocess proof | +| Phase 14 container integrity | Pre-allocation/I/O range checks, 64-byte header boundary, header/catalog/physical maximum consistency, persisted maximum, overflow-safe append, short-write detection, v0/v1 and recovery compatibility | Broad required matrix; decompression excluded | +| Phase 15 decompression | Exact expected size, absolute 4 MiB output ceiling, bounded zstd output/memory/window, identity exactness, Restore/Verify/reuse path | Broad required matrix; 4 MiB is not a container maximum | +| Phase 16 JSON fidelity | Recursive `UseNumber`, strict EOF, exact `2^53+1` and `MaxInt64` tokens for stats/inspect/simulate-GC | Integers remain JSON numbers; downstream JavaScript precision is unclaimed | +| Phase 17 SQL mutations | 70 audited: 20 hardened, 18 intentionally zero-safe, 32 already safe; SQLite/PostgreSQL affected-row semantics, rollback, and no physical GC delete after mismatch | Named PostgreSQL cardinality event; no blanket exact-one rule | +| Phase 18 CI | Fail-closed named PostgreSQL cardinality, storage/recovery, and Linux process events; required skip rejection | Existing job topology; CodeQL remains separate | + +## Backend and coordination boundaries + +The release proves selected SQLite/PostgreSQL contracts, not blanket backend +equivalence. It does not claim identical packages across both backends, every +adversarial test on SQLite, or every PostgreSQL path on every OS. + +- Linux/macOS/Windows native coordination runtime: **PROVEN** +- Production Coordinator lifecycle on Linux/macOS/Windows: **PROVEN** +- Linux independent-process contention: **PROVEN** +- Linux killed-holder release: **PROVEN** +- Linux real live-GC cross-process exclusion: **PROVEN** +- PostgreSQL dedicated advisory-session ownership: **PROVEN / CLOSED** +- macOS/Windows independent-process subprocess semantics: **NOT SEPARATELY PROVEN** +- Cross-host/distributed coordination: **NOT CLAIMED** +- Network-filesystem safety: **NOT GUARANTEED** +- FreeBSD coordination: **COMPILE-ONLY / UNSUPPORTED BACKEND** + +## Benchmark governance and BKC-016 + +Benchmark integrity is hard required. Hosted timing is advisory. Invalid or +missing evidence and evaluator failure remain blocking, but a valid historical +threshold crossing does not fail the timing-advisory job. Hard timing-regression +enforcement remains deferred to controlled infrastructure. + +BKC-016 remains **Deferred — documented**. The bounded supported-platform and +Linux process proof does not broaden the release contract to cross-host, +network-filesystem, or separately executed macOS/Windows subprocess semantics. + +## Deferred and unsupported inventory + +- **Closed by later phase:** multi-process contention; stored-path dry-run; + killed-holder, live-GC, and advisory-session items that were pending at Phase + 12 closure. +- **Intentionally deferred:** BKC-016; controlled-infrastructure hard timing; + deferred catalog APIs; SQLite-first default; repair/recovery activation; + batch-delete optimization; optional post-batch validation; structured + invariant logging; expanded repair scopes; automatic doctor repair; GC + integrity bypass. +- **Outside v1.13.11:** cross-host/distributed coordination; network filesystems; + separate macOS/Windows subprocess proof; FreeBSD runtime coordination; + blanket SQLite/PostgreSQL parity. +- **Phase 20:** final exact-head local release gate and single-PR authorization. +- **Real blocker:** none. + +## Evidence and link audit + +Every relative Markdown link in the current validation/release documents +resolves to an existing repository file. Every active tracker link to a +v1.13.11 evidence artifact exists. No duplicate or obsolete closure link needs +removal. Historical evidence retains the statements that were accurate at its +recorded phase boundary. + +## Phase 20 prerequisites + +Already satisfied: + +- all Phases 0–18 are closed with bounded evidence; +- validation and backend claim matrices are reconciled; +- benchmark and BKC-016 governance is frozen; +- active evidence links exist; and +- no unresolved product or CI blocker exists. + +Phase 20 must still: + +1. create its final gate/authorization documentation candidate; +2. verify a clean `release/v1.13.11` worktree whose local and origin heads are + identical; +3. start or reuse a healthy PostgreSQL 16 service at the documented local + endpoint; +4. execute the corrected complete local Profile A gate on that exact committed + head, including hard integrity and advisory timing policy; and +5. authorize at most one release PR only after the exact-head gate is green. + +The existing Compose PostgreSQL container may be restarted and reused after a +health check. Its existence is not validation evidence. + +## Local Phase 19 validation + +The documentation candidate must pass diff hygiene, module verification/tidy +diff, release-state fixture and real-state validation, validation-matrix audit, +local CI-enforcement audit, versioned-row-writer and smart-quote guards, and the +repository-local relative-link existence scan. Full Go/race/PostgreSQL and +adversarial reruns belong to Phase 20, not this documentation reconciliation. + +## Final Phase 19 state + +- **Phases 0–19:** COMPLETE +- **Phase 20:** NEXT +- **Release gate:** NOT STARTED +- **v1.13.11:** ACTIVE / INCOMPLETE +- **BKC-016:** Deferred — documented +- **PR/merge/tag/release authorization:** NONE + +**Phase 19 closure verdict:** READY / COMPLETE. diff --git a/docs/release/v1.13/v1.13.11-phase3-reusable-dual-backend-test-harness.md b/docs/release/v1.13/v1.13.11-phase3-reusable-dual-backend-test-harness.md new file mode 100644 index 00000000..a7af6e89 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase3-reusable-dual-backend-test-harness.md @@ -0,0 +1,45 @@ +# Coldkeep v1.13.11 Phase 3 Reusable Dual-Backend Test Harness + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** `3 — Reusable Dual-Backend Test Harness` +**Status:** Complete +**Branch:** `release/v1.13.11` + +## Outcome + +Phase 3 adds `internal/testutil/backendtest`, a test-support package for +isolated, sequential SQLite and PostgreSQL contract fixtures. It is not a +production database abstraction and does not alter normal runtime backend +selection. + +The harness creates file-backed SQLite fixtures, applies the canonical SQLite +session pragmas, and bootstraps the current schema when requested. PostgreSQL +uses one generated scratch database per backend subtest. Its lifecycle opens +and pings an admin connection, creates and bootstraps the scratch database, +then closes the tested connection, terminates remaining sessions, drops the +database, and closes the admin connection. Cleanup errors are reported by the +test rather than ignored. + +`PostgresOptional` skips only when `COLDKEEP_TEST_DB` is absent. Once that +variable is set, connection, privilege, creation, bootstrap, and cleanup +problems fail the PostgreSQL subtest. `PostgresRequired` never downgrades an +absent or unusable PostgreSQL configuration to a skip. + +## Evidence and boundary + +- Harness self-tests cover SQLite current-schema behavior, independently opened + SQLite connections, stable fixture names/capabilities, selection policy, + scratch-name validation, and cleanup ordering seams. +- The existing catalog backend-contract suite now adopts the harness without + changing its logical fixture assertions. +- Required CI has not changed. `quality` still lacks a PostgreSQL service and + `COLDKEEP_TEST_DB`, so optional catalog PostgreSQL subtests still skip there. + Phase 4 owns required-CI activation. +- This infrastructure does not establish schema, catalog, engine, mutation, or + locking parity. All Phase 2 claim classifications remain unchanged. + +## Next phase + +Phase 4 — Existing PostgreSQL-Gated Package Suite CI Activation is Next. It +must provision PostgreSQL and make the already-gated package suite execute in +required CI before the harness becomes required-CI parity evidence. diff --git a/docs/release/v1.13/v1.13.11-phase4-existing-postgresql-gated-package-suite-ci-activation.md b/docs/release/v1.13/v1.13.11-phase4-existing-postgresql-gated-package-suite-ci-activation.md new file mode 100644 index 00000000..0f2275b5 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase4-existing-postgresql-gated-package-suite-ci-activation.md @@ -0,0 +1,62 @@ +# Coldkeep v1.13.11 Phase 4 Existing PostgreSQL-Gated Package Suite CI Activation + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** `4 — Existing PostgreSQL-Gated Package Suite CI Activation` +**Status:** Complete +**Branch:** `release/v1.13.11` +**Starting commit:** `7259b9de test: add reusable dual-backend harness` + +## Prior gap and implementation + +Before Phase 4, required `quality` CI ran internal packages without a +PostgreSQL service or `COLDKEEP_TEST_DB`; PostgreSQL-gated package tests, +including the Phase 3 harness and catalog `/postgres` subtests, skipped. + +Phase 4 adds one blocking `Run required PostgreSQL internal package contracts` +step to `correctness-matrix`. It uses that job's existing PostgreSQL 16 service +and runs only in the `plain` codec matrix leg, avoiding duplicate execution of +backend-neutral package tests under the AES codec. + +The step sets the project PostgreSQL gate, auto-bootstrap, and local service +connection environment. It runs `go test -race -count=1 -json` once for the +confirmed live-gated internal packages: + +- `./internal/testutil/backendtest` +- `./internal/catalog` +- `./internal/db` +- `./internal/engine` +- `./internal/maintenance` + +`internal/storage` was inspected and is excluded: its only +`COLDKEEP_TEST_DB` reference describes a SQLite-only test rather than a live +PostgreSQL gate. + +## Execution proof and enforcement + +The workflow records JSON-lines output with `tee` and parses it after the Go +test command succeeds. The parser requires successful PostgreSQL execution of +the harness `/postgres` callback, all catalog contract `/postgres` subtests, +and an exact known gated test in each DB, engine, and maintenance package. It +fails with expected selectors, observed PostgreSQL tests, skipped tests, and a +JSON tail when evidence is absent, malformed, or incomplete. + +`scripts/audit_ci_enforcement.sh` now enforces the step placement, plain-codec +condition, PostgreSQL environment, test command, full package inventory, JSON +proof parser, and blocking failure behavior. The catalog fixture comment now +accurately names this correctness-matrix step as the required-CI execution +location. + +## Local validation and remaining evidence + +Local harness/catalog, race, vet, release-state, documentation, and CI-audit +checks validated the implementation. Local PostgreSQL was unavailable. + +Remote CI run `29729981751` completed successfully at Phase 4 commit +`0147851d8ff66ce917e7451c1d36221480f84cae`. Its successful +`correctness-matrix (plain)` job (`88311918312`) ran the package-contract step; +the JSON log recorded pass events for the harness, all six catalog `/postgres` +contracts, and the required DB, engine, and maintenance PostgreSQL tests. The +aggregate `CI Required Gate` also succeeded. + +Phase 5 is now Next. This phase activates existing tests; it does not claim +schema or catalog parity. diff --git a/docs/release/v1.13/v1.13.11-phase5-schema-bootstrap-and-migration-parity.md b/docs/release/v1.13/v1.13.11-phase5-schema-bootstrap-and-migration-parity.md new file mode 100644 index 00000000..1d5e4bff --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase5-schema-bootstrap-and-migration-parity.md @@ -0,0 +1,115 @@ +# v1.13.11 Phase 5 — Schema, Bootstrap, and Migration Parity + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** 5 +**Status:** Complete + +## Scope + +Phase 5 added outcome-oriented schema contracts in `internal/db` using the +Phase 3 `internal/testutil/backendtest` fixture. It did not change production +schema, migrations, catalog behavior, engine behavior, or runtime backend +selection. The current schema version remains **16**; no schema version 17 and +no production schema defect were introduced. + +## Schema-contract implementation and proof + +Implementation commit `2b603b7c` (`test: prove schema bootstrap and migration +contracts`) established executable SCH-001 through SCH-011 coverage wherever +production behavior is defined. SCH-012 records the transaction-boundary +limitation: SQLite post-schema migration work is transactional, while +schema-script application is not claimed atomic because production exposes no +safe injected-failure seam. + +The contracts cover empty bootstrap and current-version handling, idempotent +`EnsureSchema`, minimal schema usability, selected uniqueness and foreign-key +behavior, selected nullable/default semantics, SQLite v12 preservation, +PostgreSQL v11 auto-migration, and selected invalid or missing +schema-version states. They do not claim duplicate-version recovery, +downgrades, interrupted-migration recovery, or arbitrary partial-schema +recovery. + +Required PostgreSQL execution was proven by CI run `29803865860` at +`2b603b7c7a0b6e2174f08345a29949f2a7af7692`, correctness-matrix plain job +`88550222280`. Quality, canonical lint, and vet succeeded; the required +PostgreSQL internal-package step succeeded; and its JSON selectors confirmed +actual Phase 5 PostgreSQL SCH events rather than optional local skips. That run +is not described as fully green because its aggregate gate exposed the separate +G6 issue below. + +## Phase 5 CI-blocker remediation — G6 shared packed-block integrity + +The two preserved G6 failures from run `29803865860` and its debug rerun +blocked Phase 5 after the schema contracts had passed. Their final symptom was +a packed-block relational/encoded membership inconsistency, not a proven +physical payload-hash mismatch: reconstructed physical payload verification +remained valid while one packed-block member mapping had been removed although +the immutable payload still encoded all original members. Companion validation +then became unstable because it derived a prefix length from incomplete mutable +membership. + +Phase 5 did not introduce this storage defect; its push re-executed and exposed +an existing residual G6 risk. + +Diagnostic commit `bfe49176` (`test: capture G6 packed-block rebuild +diagnostics`) added sanitized worker lifecycle and reuse-validation traces, +support for both `chunk 1` and `chunk=1`, complete relational and encoded +membership, companion and physical-file identities, expected and actual +physical hashes, container metadata, and manifest/redaction/parsing/bounds +tests. A bounded PostgreSQL 16 reproduction ran five plain and five AES-GCM +attempts; all passed, so no production remediation was chosen solely from that +non-reproduction. + +The deterministic storage test +`TestSharedPackedBlockSingleMemberRebuildCannotLeavePartialMembership` then +proved the mechanism: a valid immutable packed block initially had matching +encoded, relational, and companion membership `[1,2]` and passed physical and +full repository verification. Rebuilding chunk `1` removed only its relational +and companion mappings while the encoded membership remained `[1,2]`; the +shared block remained, surviving companion validation became invalid, and full +verification failed. + +Corrective commit `54ecd84c` (`fix: refuse partial rebuild of shared packed +blocks`) gathers the complete candidate packed-block set inside the existing +transaction, uses backend-aware PostgreSQL row locking, and refuses an +individual rebuild when any candidate has multiple active members. The stable +`errors.Is`-classifiable error rolls the transaction back before chunk state or +mappings change. Encoded, relational, companion, container, physical-file, +offset, size, and hash state remain unchanged for shared blocks. Existing +single-member rebuild and orphan cleanup remain available. + +> Genuinely invalid shared packed blocks fail closed. Atomic whole-block repair +> remains outside this corrective scope. + +## Final exact-head CI evidence + +Corrective head `54ecd84c88cdc75622d279215f7e8f0a9de6dbb4` was validated by CI +run `29815330238`, workflow attempt 2. Quality/lint, PostgreSQL correctness +contracts, both correctness matrices, both integration-stress and long-run +legs, both smoke legs, legacy compatibility, and Ubuntu/macOS/Windows +cross-platform validation succeeded. Both adversarial jobs passed deterministic +PostgreSQL G6 and full G1–G17 validation for plain and AES-GCM; G6 failure +diagnostics were skipped because no failure occurred. The zstd benchmark matrix +succeeded. + +The first exact-head attempt failed only the workers=4 uncompressed +snapshot-creation benchmark threshold. One authorized rerun of only that failed +benchmark job at the identical corrective SHA succeeded. No code, benchmark +baseline, workflow, or configuration changed between attempts. The rerun, +benchmark job `89121415944`, also caused CI Required Gate job `89121652949` to +succeed. This is recorded as a transient benchmark anomaly resolved by one +authorized same-SHA retry; it is not erased, not treated as a confirmed +production regression, and does not prove zero performance variance. + +## Conservative compatibility result and next phase + +Selected contracts now have executable SQLite and required PostgreSQL evidence: +empty/current bootstrap, schema version 16, idempotency, minimal usability, +selected uniqueness and foreign keys, selected nullable/default semantics, +SQLite v12 preservation, PostgreSQL v11 auto-migration, and selected invalid +metadata behavior. Broad schema equivalence remains intentionally conservative +because historical migration starting states are asymmetric and the full schema +surface is not a shared contract. + +Phase 5 is Complete. Phase 6 — Implemented Catalog Contract Parity — is the +sole Next phase. No Phase 6 implementation has started. diff --git a/docs/release/v1.13/v1.13.11-phase6-implemented-catalog-contract-parity.md b/docs/release/v1.13/v1.13.11-phase6-implemented-catalog-contract-parity.md new file mode 100644 index 00000000..0b52d8bf --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase6-implemented-catalog-contract-parity.md @@ -0,0 +1,86 @@ +# v1.13.11 Phase 6 — Implemented Catalog Contract Parity + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Phase:** 6 +**Starting commit:** `460ff671` +**Status:** Complete + +## Scope and method inventory + +Phase 6 extends the existing Phase 3 dual-backend fixture and the six catalog +contract tests. The implemented methods under proof are `FindLogicalFile`, +`FindPhysicalFilesForLogicalFile`, `FindSnapshot`, `ListSnapshots`, and +`LoadReachabilityRoots`. `LoadSnapshotGraph`, `LoadChunkPlacements`, +`LoadRestorePlanMetadata`, and `LoadGCPlanMetadata` remain explicit, +non-mutating `ErrNotImplemented` boundaries owned by v1.13.12; this phase does +not implement them. + +The backend-neutral fixture uses deterministic identifiers, UTC timestamps, +paths, labels, and status values. It includes a supported large `int64`, null +and non-null physical metadata, true and false booleans, root/child/null-label +snapshots, equal snapshot timestamps, duplicate current and snapshot +reachability inputs, current-only/snapshot-only/both roots, an unreferenced +file, and a processing current reference. SQLite and PostgreSQL receive the +same fixture meaning through `$n` parameters. + +## Executable contracts + +| ID | Contract | Evidence | +| --- | --- | --- | +| CAT-001 | Logical file lookup | Existing/missing results, exact fields, supported large `int64`, repeated reads, cancellation error behavior, and non-mutation. | +| CAT-002 | Physical file lookup | Empty missing result, `path` ordering, nullable `mtime`, nullable mode normalization, true/false `is_metadata_complete`, and repeated reads. | +| CAT-003 | Snapshot lookup | Missing/root/child/null-label records, parent identity, type, timestamp instant, and repeated reads. | +| CAT-004 | Snapshot listing and filtering | Newest-first order, equal-time ID tie order, type and literal substring filters, inclusive Since/Until boundaries, combined filters, limit/zero/negative behavior, and empty results. | +| CAT-005 | Reachability roots | Distinct current/snapshot sets, duplicate elimination, current-only/snapshot-only/both roots, processing physical references, unreferenced exclusion, and independent maps. | +| CAT-006 | Deferred method boundary | All four deferred methods return `errors.Is(err, catalog.ErrNotImplemented)`, return nil results, and do not mutate state. | +| CAT-007 | Error and context behavior | A pre-cancelled context produces errors rather than missing results for every implemented read method and leaves state unchanged. Raw backend error strings are not asserted. | + +The public snapshot model deliberately represents nullable `label` and +`parent_id` as empty strings. SQLite/PostgreSQL timestamp representations are +compared as the same UTC instant. SQL syntax, physical types, query plans, and +raw database error text are intentionally not parity criteria. + +## Files and local validation + +`internal/catalog/backend_contract_test.go` now strengthens the six existing +top-level tests, so the Phase 4 CI JSON selectors continue to identify every +PostgreSQL contract execution without workflow or audit changes. The local +SQLite suite and race suite passed, as did nearby `internal/db` and +`internal/testutil/backendtest` ordinary and race suites. + +The first exact-head Phase 6 CI attempt (`29982838592`, +`c03c42bdb7dcaa768787167725b5e1cafc8f33c6`) proved a PostgreSQL CAT-004 +defect: equal `created_at` values had no SQL secondary key, so PostgreSQL +returned a different tie order than SQLite. `ListSnapshots` now orders by +`created_at DESC, id DESC` in both limited and unbounded queries. This is a +narrow public-ordering correction; it changes no schema, filter, or deferred +API behavior. It still requires exact-head PostgreSQL proof after the +correction is committed and pushed. + +No PostgreSQL container was available locally: Docker daemon access showed no +running `coldkeep-g6-postgres` container. PostgreSQL child subtests therefore +remain unexecuted locally under the harness's optional policy. This limitation +was closed by required CI rather than reclassified as local parity evidence. + +## Classification and remaining proof + +The corrected exact-head CI run `29983479388` completed successfully at +`db12c3d24e5272396a75c3bf773020c2876ab86d`. Its plain +`correctness-matrix` job `89130181273` recorded PostgreSQL pass events for +CAT-001 through CAT-006, including +`TestCatalogContractListSnapshotsAcrossBackends/postgres`. The corresponding +SQLite and PostgreSQL fixture assertions establish the scoped implemented +catalog contracts: the five implemented methods, placeholder binding, +nullability normalization, deterministic ordering, filtering, limits, and +cancelled-context errors. The `quality` job `89130181335`, both adversarial +jobs (`89132028503` and `89132028518`), and `CI Required Gate` +`89132978498` also succeeded. + +BKC-007 and BKC-009 are therefore `Equivalent — proven` only for those +executed CAT contracts. BKC-008 remains `Deferred — documented`; no deferred +API was activated. CAT-004's missing deterministic secondary ordering was the +only Phase 6 production defect; `db12c3d2` corrects it with +`created_at DESC, id DESC` without changing schema, filter, or deferred API +behavior. + +Phase 6 is Complete. Phase 7 — Engine Read-Side Backend Parity — is Next. diff --git a/docs/release/v1.13/v1.13.11-phase7-engine-read-side-backend-parity.md b/docs/release/v1.13/v1.13.11-phase7-engine-read-side-backend-parity.md new file mode 100644 index 00000000..903917ce --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase7-engine-read-side-backend-parity.md @@ -0,0 +1,85 @@ +# Coldkeep v1.13.11 Phase 7 — Engine Read-Side Backend Parity + +**Status:** Complete +**Branch:** `release/v1.13.11` +**Implementation commit:** `3e8c98df` +**Corrective commit:** `313d0069` + +## Scope + +Phase 7 adds one backend-neutral engine fixture and four sequential +SQLite/PostgreSQL contract tests. It covers public read-side results for +`Stats`, `Inspect`, `Verify`, and explicit-ID snapshot list/show/stats/diff +views. It does not define selector resolution, invalid-regex behavior, tree +presentation, repeated CLI selectors, mutations, GC, or locking. + +## Implemented contracts + +- `ENG-R-001` checks repository summaries, supported inspection, container + inclusion, relation ordering, missing entities, and the distinct + `observability.ErrUnsupportedEntity` physical-file boundary. +- `ENG-R-002` checks stable `created_at DESC, id DESC` snapshot-list order, + basic catalog-backed filters, nullable metadata, explicit-ID show/stats/diff + results, lineage, and stable path ordering. +- `ENG-R-003` checks clean fast, standard, full, and deep verification, + container-backed fixture integrity, and one controlled structural + inconsistency without comparing raw driver errors. +- `ENG-R-004` checks pre-cancelled contexts, missing/invalid/unsupported + result distinctions, repeated-read stability, and non-mutation. + +The fixture writes two deterministic payloads through the normal storage +writer, finalizes its container, then seeds deterministic snapshot graph data +with backend-neutral parameters. A test-only fingerprint records relational +state, retry/pin/status values, relevant timestamps, and container file +paths/sizes/content hashes before and after read operations. + +The pre-cancelled `Verify` regression initially completed verification despite +the cancelled context. `DefaultEngine.Verify` now returns `ctx.Err()` before +verification dispatch. Non-cancelled verification behavior is unchanged. + +## First exact-head CI result and blocker + +The existing plain correctness-matrix package-contract step retains its single +`internal/engine` invocation and now requires `/postgres` pass events for all +four ENG-R tests. Exact-head CI run `29990126227` executed and passed all four +PostgreSQL subtests at `3e8c98df`, but the run was not green. SQLite +`ENG-R-003` held the only harness connection with the outer packed-container +rows while deep verification attempted a nested `storage_blocks` query. The +operation timed out after five minutes, failing quality, plain correctness, +and the aggregate required gate. CodeQL run `29990126253` succeeded separately. + +The local remediation materializes the eligible container metadata and +explicitly closes the outer rows before querying any container's packed +blocks. `TestVerifySystemDeepPackedSQLiteSingleConnection` reproduces the old +failure with `MaxOpenConns(1)` and a 250 ms child-process operation timeout; +the corrected path passes without increasing connection limits or changing +byte-level verification. + +## Corrective exact-head CI evidence + +Corrective commit `313d0069` passed exact-head CI run `29993172886` at +`313d0069ec124821d2d82d1493812fbdc8e9d451`. The `quality` job +`89160468816`, plain correctness job `89160468856`, both correctness codec +legs, integration, smoke, benchmark, compatibility, cross-platform, and both +adversarial jobs succeeded. The aggregate `CI Required Gate` job +`89165096957` also succeeded. + +The plain correctness job recorded pass events for all required PostgreSQL +contracts: + +- `TestEngineReadStatsAndInspectAcrossBackends/postgres`; +- `TestEngineReadSnapshotViewsAcrossBackends/postgres`; +- `TestEngineReadVerifyAcrossBackends/postgres`; +- `TestEngineReadContextAndErrorsAcrossBackends/postgres`. + +The successful quality job includes the formerly blocked SQLite +`ENG-R-003` package path. The bounded +`TestVerifySystemDeepPackedSQLiteSingleConnection` regression, with its +single connection and 250 ms child-operation timeout, remains the direct +local proof that deep verification no longer holds the outer rows while +querying packed blocks. + +Phase 7 is complete. BKC-010 is `Equivalent — proven` only for the tested +Stats, Inspect, Verify, context/error, and non-mutation contracts. BKC-011 +remains `Separate evidence — no parity proof`: explicit-ID snapshot views are +proven, but selector/query semantics belong to Phase 8, which is now Next. diff --git a/docs/release/v1.13/v1.13.11-phase8-snapshot-selector-determinism-closure.md b/docs/release/v1.13/v1.13.11-phase8-snapshot-selector-determinism-closure.md new file mode 100644 index 00000000..29f28661 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase8-snapshot-selector-determinism-closure.md @@ -0,0 +1,49 @@ +# Coldkeep v1.13.11 Phase 8 — Snapshot Selector Determinism Closure + +**Status:** Complete +**Branch:** `release/v1.13.11` + +## Implemented scope + +- `TestEngineSnapshotSelectorsAcrossBackends` proves equal-time snapshot list + ordering, inclusive time bounds, label filtering, file-query intersection, + deterministic repeated reads, and filtered diff ordering. +- `TestEngineSnapshotSelectorErrorsAcrossBackends` proves invalid direct-engine + regex rejection, pre-cancelled selection, missing snapshot classification, + and database/container non-mutation. +- Engine query conversion now returns an invalid-request error for malformed + regexes instead of silently dropping the regex filter. +- CLI diff retains its complete snapshot-domain query, including repeated + `--path` and `--prefix` selectors, rather than selecting an arbitrary map + member through the narrower engine seam. +- CLI tree shaping keeps equal-time ID ordering, treats missing parents as + roots, breaks cycles, and emits duplicate metadata IDs only once. + +## Exact-head CI closure evidence + +- Implementation commit: `bcae3576429359747b8edf809c9656bda22001fb`. +- Exact-head CI run `30109561344` completed successfully. Quality job + `89535269631`, plain correctness job `89535269535`, and aggregate + `CI Required Gate` job `89540053230` all passed. +- The plain-codec internal-package invocation recorded both required events: + `TestEngineSnapshotSelectorsAcrossBackends/postgres` passed in 0.27s and + `TestEngineSnapshotSelectorErrorsAcrossBackends/postgres` passed in 0.23s. +- The same run also passed both correctness codecs, adversarial, stress, + long-run, smoke, compatibility, benchmark, and cross-platform jobs. + +## Confirmed correction + +The Phase 8 tests confirmed that malformed direct-engine regexes were silently +dropped during conversion. The narrow conversion path now returns and +propagates a stable invalid-request error through snapshot show and detailed +diff. No selector syntax or mutation behavior changed. + +## Boundaries + +Phase 8 does not establish snapshot mutation/restore selector parity, a +latest/tag/batch-ID selector, label case-insensitivity, wildcard escaping, +locking, schema, or final release-gate completion. BKC-011 is +`Equivalent — proven` only for the implemented snapshot list/show/stats/diff +selector and CLI tree-presentation contracts described above; mutations, +restore selection, latest/tags/batch IDs, label case behavior, wildcard +escaping, and later-phase work remain excluded. diff --git a/docs/release/v1.13/v1.13.11-phase9-engine-mutation-backend-parity.md b/docs/release/v1.13/v1.13.11-phase9-engine-mutation-backend-parity.md new file mode 100644 index 00000000..2dfac317 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-phase9-engine-mutation-backend-parity.md @@ -0,0 +1,122 @@ +# Coldkeep v1.13.11 Phase 9 — Engine Mutation Backend Parity + +**Status:** Complete +**Branch:** `release/v1.13.11` + +## Exact-head CI closure + +Implementation `848e579b28eb49222378bf908e12533d1972c359` passed required +CI run `30114444798`. Quality job `89551564782`, plain correctness job +`89551564893`, both codec correctness jobs, stress, long-run, adversarial, +smoke, compatibility, benchmark, and cross-platform jobs succeeded; aggregate +required-gate job `89555865026` was green. + +The plain correctness job recorded passing PostgreSQL events for all required +contracts: + +- `TestEngineMutationStoreRemoveAcrossBackends/postgres` +- `TestEngineMutationSnapshotLifecycleAcrossBackends/postgres` +- `TestEngineMutationRestoreAcrossBackends/postgres` +- `TestEngineMutationErrorsAcrossBackends/postgres` +- `TestEngineGCDryRunAcrossBackends/postgres` + +## Implemented scope + +Phase 9 adds five shared `backendtest.ForEach` contracts: + +- `TestEngineMutationStoreRemoveAcrossBackends` covers single-file plain-codec + Store, content deduplication, physical-path replacement, empty files, + stored-path unlink, by-ID dry-run/live Remove, deterministic batch shape, + and retained container payloads. +- `TestEngineMutationSnapshotLifecycleAcrossBackends` covers explicit full, + parented full, and overlapping partial snapshot creation, deterministic + membership, read-only delete preview, metadata-only execute, and child + parent clearing. +- `TestEngineMutationRestoreAcrossBackends` covers by-ID, stored-path, and + selected snapshot restore output bytes, hashes, paths, ordering, repository + neutrality, and zero final chunk pins. +- `TestEngineMutationErrorsAcrossBackends` covers pre-cancelled requests, + deterministic Store validation and unsupported classification, + snapshot-retained removal, snapshot validation/not-found failures, + restore collision and partial-batch behavior, and transactional stored-path + ref-count mismatch rollback. +- `TestEngineGCDryRunAcrossBackends` covers a fixed dead sealed-container plan, + current-and-snapshot reachability counts, ordered filenames, repeated result + stability, pre-cancelled execution, and database/container non-mutation. + +The fixture uses production `LocalWriter` storage with explicit `plain` codec +and finalizes every writer before restore, GC, or container-integrity checks. +Direct SQL is limited to stable fixture normalization, deterministic corruption +injection, and immutable GC-plan data. + +## Semantic integrity evidence + +The shared helper captures typed, sorted semantic state for logical files, +physical mappings, recipes, chunks and pin counts, legacy and packed placement +edges, containers, snapshots, standalone normalized snapshot paths, +membership, table counts, and container file size/SHA-256 manifests. Restore +destinations are verified as complete sorted manifests by relative path, byte +length, content hash, and exact bytes. Store/remove assertions also verify a +non-empty recipe and the expected chunk live-reference transition. + +Generated numeric IDs, generated timestamps, container names produced by the +writer, and raw cross-backend container bytes are not compared. Missing graph +edges, changed logical hashes, changed status/count fields, extra rows, payload +rewrites, or residual pins remain visible failures. + +## Local validation + +- Focused SQLite contracts passed with `-count=1`. +- The same five contracts passed ten repeated executions with `-count=10`. +- The same five contracts passed with `-race -count=1`. +- Full nearby Engine, storage, snapshot, maintenance, container, batch, + catalog, and CLI package tests passed both normally and under the race + detector; no package was excluded. +- Focused storage, Engine, and snapshot restore race suites passed. +- `go vet ./...` and canonical `golangci-lint run ./...` passed with zero + issues. +- Release-state validator tests and real-state validation, smart-quote, + validation-matrix, versioned-row-writer, local CI-enforcement, and + `git diff --check` guards passed. +- PostgreSQL subtests skip locally because `COLDKEEP_TEST_DB` is unset. +- The combined local/remote CI audit cannot inspect GitHub policy because the + installed `gh` CLI is unauthenticated; its supported `--local-only` mode + passes. Exact-head required CI remains the remote execution proof. + +The first diagnostic run exposed only fixture assumptions: production Store +canonicalizes physical paths, by-ID restore uses the logical original basename, +and a deterministic dead GC candidate must be seeded as immutable plan data. +After aligning the fixture with those established contracts, no backend +production divergence remained. No production file was changed. + +## CI proof boundary + +The existing plain-codec internal-package invocation is unchanged. Its JSON +event parser and the matching local CI audit now require: + +- `TestEngineMutationStoreRemoveAcrossBackends/postgres` +- `TestEngineMutationSnapshotLifecycleAcrossBackends/postgres` +- `TestEngineMutationRestoreAcrossBackends/postgres` +- `TestEngineMutationErrorsAcrossBackends/postgres` +- `TestEngineGCDryRunAcrossBackends/postgres` + +Missing, skipped, failed, or absent pass events remain fatal to the parser. +These requirements were met by run `30114444798` at `848e579b`. + +## Final BKC classification + +- BKC-012 is `Equivalent — proven` only for the tested active, uncontended + Store/Remove/Restore and snapshot mutation outcomes. +- BKC-013 is `Equivalent — proven` only for the tested GC dry-run plan, + result ordering, and non-mutation behavior. +- BKC-014 remains `Backend-specific — proven`. +- BKC-015 remains `Unproven`. + +## Boundaries + +Phase 9 does not cover recursive Engine Store, CLI `store-folder`, dormant +Store tags/metrics, GC workers, mid-operation cancellation, concurrent +writers, row locks, isolation, advisory/process locks, live GC, repair or +recovery, schema/format/codec changes, encrypted container equality, metadata +or path behavior outside the exercised combinations, or Phase 10 and later +work. Batch operations remain sequential and are not globally atomic. diff --git a/docs/release/v1.13/v1.13.11-release-gate.md b/docs/release/v1.13/v1.13.11-release-gate.md new file mode 100644 index 00000000..be4beef1 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-release-gate.md @@ -0,0 +1,227 @@ +# Coldkeep v1.13.11 Release Gate + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Status:** Passed — candidate preparation only; exact-head execution required +**Branch:** `release/v1.13.11` +**Starting baseline:** `423c57815580c39bee4f79ecd81570e9cfa9d273` +**Source version:** `1.13.11` +**Gate status:** Ready for exact-head execution + +## Exact candidate contract + +The immutable release candidate is the commit containing this gate record. +This document intentionally does not guess or embed its own final SHA. The +candidate SHA is captured from Git after commit and retained with external +execution evidence. + +Phase 20's tracked work is complete when this prospective record and the +pre-release tracker state are committed. Operational pull-request authorization +exists only after this exact candidate passes candidate-head CI, `CI Required +Gate`, CodeQL, and the complete clean local Profile A gate. Runtime results are +kept outside the repository so the validation cannot invalidate its own head. + +After the exact-head gate starts, no tracked edit, amend, rebase, merge, or new +commit is permitted. Any tracked change creates a new candidate and requires +fresh hosted proof plus a complete local gate rerun from the beginning. + +## Completed preliminary evidence + +- Phase 1 activated source and reusable release-checklist identity to `1.13.11`. +- Local human and JSON version commands report `1.13.11`. +- The release-state validator passes for the development lifecycle. +- Phase 2 classified current SQLite/PostgreSQL claims and required-CI evidence; + it did not establish backend parity or complete the release gate. +- Phase 3 added reusable SQLite/optional-PostgreSQL package-test infrastructure + and adopted it for catalog contracts. Required-CI PostgreSQL execution and + all parity proof remain pending Phase 4 and later phases. +- Phase 4 now configures the blocking plain-codec correctness-matrix package + contract step and JSON execution proof. Required-CI run `29729981751` at + `0147851d` proved the package pass events and aggregate gate success. +- Phase 5 completed selected schema/bootstrap/migration contract proof, + including required PostgreSQL SCH execution, and recorded the separate G6 + shared packed-block fail-closed remediation. Final exact-head CI passed after + one authorized same-SHA retry of a transient workers=4 uncompressed benchmark + anomaly. Broad schema parity and automatic whole-block recovery are not + claimed. +- Phase 6 completed scoped catalog parity proof: correction `db12c3d2` passed + exact-head CI run `29983479388`; its plain correctness job recorded all six + PostgreSQL catalog-contract pass events, and `CI Required Gate` succeeded. + Deferred catalog APIs remain unavailable, and the release gate is not + complete. +- Phase 7 completed shared engine read-side contracts and required PostgreSQL + JSON proof for Stats, Inspect, Verify, and explicit-ID snapshot views. + Corrective exact-head CI run `29993172886` at `313d0069` passed quality, + plain correctness job `89160468856`, both adversarial legs, and aggregate + required-gate job `89165096957`. BKC-010 is equivalently proven only within + its tested scope; selector/query parity remains Phase 8 work. +- Phase 8 completed focused selector/error parity at implementation + `bcae3576429359747b8edf809c9656bda22001fb`. Exact-head CI run `30109561344` + passed quality job `89535269631`, plain correctness job `89535269535`, both + required PostgreSQL selector events, and aggregate required-gate job + `89540053230`. BKC-011 is equivalently proven only for the scoped + list/show/stats/diff selector and tree-presentation contracts. +- Phase 9 completed scoped mutation and GC dry-run parity at implementation + `848e579b28eb49222378bf908e12533d1972c359`. Exact-head CI run + `30114444798` passed quality job `89551564782`, all five required PostgreSQL + events in plain correctness job `89551564893`, both codec correctness legs, + stress, long-run, adversarial, smoke, compatibility, benchmark, and + cross-platform jobs, and aggregate required-gate job `89555865026`. + BKC-012/013 are equivalently proven only within their documented Phase 9 + active, uncontended mutation and GC dry-run scope. +- Phase 10 implementation `ad82c959` passed exact-head CI run `30148670910`: + all five PostgreSQL events passed in plain job `89655223183`; successful + same-head benchmark rerun `89656813012` resolved the first transient + uncompressed benchmark variance; and required gate `89656972706` passed. + BKC-003 and BKC-015 are backend-specific — proven only within their + documented boundaries. No production code changed. +- Phase 11 completes the exclusive-only repository coordination + contract: canonical container identity, recovery-safe namespace, stable + errors, diagnostic owner metadata, lease lifecycle, and command policy are + covered by fake-based tests. No OS lock, CLI acquisition, workflow selector, + subprocess proof, advisory-session correction, or SQLite live-GC change is + included. Accepted exact-head CI and CodeQL evidence is recorded in + [the Phase 11 closure evidence](v1.13.11-phase11-closure.md); BKC-016 + remains `Deferred — documented`. +- Phase 11 benchmark investigation and binary-identical run `30696834430` + accepted the bounded-v2 functional evidence architecture and rejected both + historical absolute and hosted paired timing as hard 5% endpoints. Required + CI now separates hard candidate integrity from hosted timing advice. Frozen + v1.9 measurements and thresholds remain unchanged and advisory only; hard + performance enforcement is deferred to controlled infrastructure. +- The benchmark governance implementation did not itself advance Phase 12. + Exact-head + local validation, four integrity passes, successful advisory evaluation, + complete artifacts, green CI/CodeQL, and closure evidence are recorded in + [the Phase 11 closure evidence](v1.13.11-phase11-closure.md). BKC-016 remains + `Deferred — documented`. +- Phase 12 implements the safe coordination namespace, diagnostic metadata, + process-global reservation, native Unix and Windows locking, production + Coordinator and CLI ordering, stable error mappings, direct-caller contract, + and Restore/Verify/GC outer-Lease coverage. Its bounded closure and remaining + proof limits are recorded in + [the Phase 12 closure evidence](v1.13.11-phase12-closure.md). +- Initial Phase 12G CI run `31255838594` proved native runtime on Linux, macOS, + and Windows but failed Required Gate job `93100650890` because historical G6 + tests expected overlapping stores to succeed. That run remains failed while + its native evidence remains valid. +- Phase 13A commit `cf755305` changed adversarial tests only. Reconciliation + run `31258313120` passed both adversarial codec jobs, all three native jobs, + Required Gate job `93106823979`, and independent CodeQL run `31258313118`. + It proves Linux independent-process fail-fast contention without weakening + production coordination. +- Phase 13B implementation `5bdaa443` and selector-only Staticcheck correction + `4aa2e754` prove Linux killed-holder release and immediate reacquisition + without application cleanup, sleep, or retry. Final-head CI run + `31308921622`, Required Gate job `93235437532`, and CodeQL run + `31308921567` passed. +- Phase 13C implementations `da78614a` and `54b1a04c` pin PostgreSQL GC + advisory ownership to one dedicated session and prove real Linux live-GC + cross-process exclusion. Implementation CI `31311628561`, Required Gate + `93242095156`, and CodeQL `31311628559` passed. Evidence-head CI + `31312520061`, Required Gate `93244412826`, and CodeQL `31312520060` also + passed. +- Aggregate Phase 13 closure is recorded in + [the Phase 13 closure evidence](v1.13.11-phase13-closure.md). Native and + production Coordinator runtime are proven on Linux, macOS, and Windows; + representative independent-process, killed-holder, and real live-GC process + semantics are proven on Linux. +- Phase 14 final implementation head `a576b19d` validates outer container + ranges before allocation/read, enforces supported-header/catalog/physical + maximum consistency, preserves persisted maxima through packed reads and + recovery, detects header short writes, and makes append capacity arithmetic + overflow-safe. Final-head CI run `31319918720`, Required Gate job + `93264606698`, and CodeQL run `31319918730` passed. The bounded evidence is + recorded in + [the Phase 14 evidence](v1.13.11-phase14-container-range-header-consistency.md). + BKC-016 remains + `Deferred — documented`, and the overall release gate remains incomplete. +- Phase 15 implementation head `4213f3c` enforces the exact identity/zstd + decompression contract before decoder creation or output allocation, caps + aggregate output and zstd decoder memory/window resources at 4 MiB, and + carries the shared bound through Restore, system Verify, and Store semantic + reuse. Exact-head CI run `31326930293`, Required Gate job `93281016993`, and + CodeQL run `31326930268` passed. The bounded evidence is recorded in + [the Phase 15 evidence](v1.13.11-phase15-bounded-decompression.md). The + overall release gate remains incomplete. +- Phase 16 implementation head `5dcf004` replaces the affected generic + stats/inspect/simulate-GC decode with recursive `json.Number` handling and + requires EOF after the single internally marshaled value. Exact large signed + integer tokens now survive nested maps and arrays while remaining JSON + numbers. Exact-head CI run `31332028771`, Required Gate job `93293724282`, + and CodeQL run `31332028770` passed. The bounded evidence is recorded in + [the Phase 16 evidence](v1.13.11-phase16-json-integer-fidelity.md). The + overall release gate remains incomplete. +- Phase 17 implementation head `0c50113` inventories all 70 non-DDL production + mutations and hardens M17-001 through M17-020 with the shared internal + affected-row sentinel and validator. Required mismatches now roll back before + commit, GC mismatch cannot remove a physical file, and zero-safe cleanup, + recovery, CAS, upsert, bulk, and GC semantics remain unchanged. Exact-head + CI run `31867193430`, Required Gate job `94972031158`, and CodeQL run + `31867193369` passed. The bounded evidence is recorded in + [the Phase 17 evidence](v1.13.11-phase17-fail-closed-sql-mutations.md). Phase + 18 is complete. It requires selected PostgreSQL mutation-cardinality, + storage/recovery, and Linux independent-process, killed-holder, and live-GC + execution events to pass without skips in the existing required jobs. Final + implementation head `eaa5896` passed CI run `31872189672`, Required Gate job + `94984536507`, and CodeQL run `31872189661`; the evidence is recorded in + [the Phase 18 evidence](v1.13.11-phase18-required-backend-coordination-ci.md). + Closure head `9c1fa524` then passed CI run `31873436272`, Required Gate job + `94987546529`, CodeQL run `31873436304`, and CodeQL Aggregate job + `94985540614`. +- Phase 19 completed the aggregate validation-matrix, backend-claim, + reusable-checklist, tracker, proof-boundary, deferred-item, and evidence-link + reconciliation. Its documentation-only decision is recorded in + [the Phase 19 evidence](v1.13.11-phase19-validation-evidence-reconciliation.md). + Phase 20 now freezes the prospective exact-head candidate contract and + pre-release state. Candidate-head hosted and complete local execution remain + mandatory before operational PR authorization. BKC-016 remains + `Deferred — documented`. + +## Required evidence + +- Local profiles: complete documented local correctness, race, and release + profiles at the exact release head. +- Dual backend: executable SQLite and PostgreSQL contract, schema, migration, + read, mutation, unsupported-behavior, and CI evidence. +- Multi-process: same-host coordination, startup recovery, contention, and + live-GC barrier evidence. +- Cross-platform: native required CI evidence for supported platforms. +- Adversarial and long-run: required safety, contention, decompression, SQL, + and selector adversarial/long-run profiles. +- Smoke and benchmarks: documented smoke evidence, hard four-profile candidate + integrity, and valid four-profile hosted-timing advisory evaluation. + Historical timing threshold crossings remain visible but are not hard + regression authority. +- Static analysis and CodeQL: required static-analysis, CodeQL, and CI + enforcement evidence. Codacy is advisory signal if available, not release + authority. + +## Release controls + +Validation must be performed at the exact head proposed for merge. One PR is +authorized only after Phase 20 and this full gate are green. Merge, main CI, +annotated tag, tag CI, publication, and release-branch deletion are distinct +post-local evidence and must be recorded only when independently available. + +The authorized pull-request source is `release/v1.13.11`; its target is +`main`. Phase 20 authorizes at most that one pull request and does not create +it. If `main` advances, the candidate remains immutable; any update operation +that changes the release-branch SHA invalidates authorization and restarts the +complete candidate process. + +## Final verdict + +`PRE-RELEASE CANDIDATE PREPARED` — operational PR authorization remains +conditional on successful candidate-head hosted validation and the complete +clean local exact-head Profile A gate. + +## Post-release placeholders + +| Evidence | Status | +| --- | --- | +| Merge | Not started | +| Main CI | Not started | +| Annotated tag | Not started | +| Tag CI | Not started | +| Publication | Not started | +| Release-branch deletion | Not started | diff --git a/docs/release/v1.13/v1.13.11-release-state-validator-contract.md b/docs/release/v1.13/v1.13.11-release-state-validator-contract.md new file mode 100644 index 00000000..b23124a8 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-release-state-validator-contract.md @@ -0,0 +1,19 @@ +# Coldkeep v1.13.11 Release-State Validator Contract + +## Applicability + +The release-state validator implementation and CKRS rule set remain unchanged. +Its active version-specific inputs for v1.13.11 are the source version, version +test, reusable pre-release checklist, changelog, root and release READMEs, +v1.13.11 trackers, this applicability record, and the reconciliation record. + +Phase 0 intentionally produced a temporary branch/version lifecycle mismatch: +the active branch and documents named v1.13.11 while source identity remained +1.13.10. Phase 1 resolves that mismatch by updating source, test, checklist, +changelog, and tracker identity. It does not extend, redesign, or replace CKRS. + +The expected development result is: + +```json +{"status":"ok","validator":"coldkeep-release-state","state":"development","active_version":"1.13.11","violations":[],"error":null} +``` diff --git a/docs/release/v1.13/v1.13.11-release-train-reconciliation.md b/docs/release/v1.13/v1.13.11-release-train-reconciliation.md new file mode 100644 index 00000000..ecda41ae --- /dev/null +++ b/docs/release/v1.13/v1.13.11-release-train-reconciliation.md @@ -0,0 +1,18 @@ +# Coldkeep v1.13.11 Release-Train Reconciliation + +## Purpose + +This applicability record carries the Phase 0 closure correction into the +active v1.13.11 release-state contract. The detailed correction is recorded in +[v1.13.11-phase0-post-release-closure-correction-and-baseline.md](v1.13.11-phase0-post-release-closure-correction-and-baseline.md). + +v1.13.10 remains the released baseline. v1.13.11–v1.13.13 are active again: + +- v1.13.11 — Safety and Backend Compatibility Gate Closure. +- v1.13.12 — Engine and Catalog Completion. +- v1.13.13 — Final v1.x and v2 Handoff Gate. + +The authoritative release train remains `v1.13.x-release-train.md`. Phase 1 +changes only release identity; it does not implement backend, safety, or +architecture work. Phase 2 — Backend Compatibility Claim Matrix is the next +executable planning phase. diff --git a/docs/release/v1.13/v1.13.11-scope.md b/docs/release/v1.13/v1.13.11-scope.md new file mode 100644 index 00000000..2aebadcd --- /dev/null +++ b/docs/release/v1.13/v1.13.11-scope.md @@ -0,0 +1,125 @@ +# Coldkeep v1.13.11 Scope + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Status:** Ready for release +**Branch:** `release/v1.13.11` +**Starting baseline:** `423c57815580c39bee4f79ecd81570e9cfa9d273` + +## Current phase + +Phases 0–20 are Complete. Phase 12 implements the Phase 11 repository-wide, +exclusive, fail-fast, non-reentrant coordination contract for same-host local +filesystems and proves native runtime plus production Coordinator lifecycle on +Linux, macOS, and Windows. + +Phase 13 closes the required multi-process evidence. Phase 13A proves +deterministic Linux independent-process contention while preserving and +strengthening G6 integrity. Phase 13B proves kernel-owned killed-holder release +and immediate reacquisition without application cleanup, sleep, or retry. +Phase 13C proves real Linux live-GC cross-process exclusion and corrects +PostgreSQL advisory locking to retain one dedicated owning session throughout +GC, require a true unlock, discard uncertain sessions, preserve cleanup +errors, and reject a single-connection pool. + +The closure is recorded in +[v1.13.11-phase13-closure.md](v1.13.11-phase13-closure.md). macOS and Windows +native and production Coordinator runtime remain proven; their independent +process, killed-holder, and live-GC subprocess semantics are not separately +proven and are outside the v1.13.11 acceptance contract. Cross-host +coordination is unsupported, distributed coordination is not claimed, and +network-filesystem safety is not guaranteed. BKC-016 remains +`Deferred — documented`. + +Phase 14 validates outer container ranges before allocation or filesystem I/O, +separates payload reads from the fixed header, enforces supported-header, +catalog-maximum, and physical-size consistency, and makes the final append gate +overflow-safe. Packed readers use each container's persisted catalog maximum, +preserving valid repositories across configuration changes. Supported v0/v1 +bytes remain compatible; there is no format, schema, migration, public API, +coordination, decompression, or JSON change. The evidence is recorded in +[v1.13.11-phase14-container-range-header-consistency.md](v1.13.11-phase14-container-range-header-consistency.md). + +Phase 15 validates every production decompression expectation against a fixed +4 MiB ceiling before decoder creation or output allocation. Identity preserves +its no-copy path while enforcing exact size. Zstd preserves per-call +whole-buffer decoding while bounding both aggregate output and decoder +memory/window resources; short, long, malformed, truncated, and concatenated +input fails through the established typed error contract. Restore, system +Verify, and Store semantic reuse inherit the same shared verification path. +Released packed-writer output remains below the ceiling, and there is no +format, schema, migration, public API, CLI, encryption, hash, container, or JSON +change. The evidence is recorded in +[v1.13.11-phase15-bounded-decompression.md](v1.13.11-phase15-bounded-decompression.md). + +Phase 16 replaces the affected generic JSON decode with recursive +`json.Number` handling and requires EOF after one internally marshaled value. +Stats, inspect, and simulate-GC therefore preserve exact signed `int64` tokens, +including nested maps and arrays, while integers remain JSON numbers. Envelope +v1.7 fields and nesting, ordinary floats, errors, stdout/stderr routing, +coordinated spool replay, schemas, storage formats, public APIs, dependencies, +and workflows remain unchanged. The evidence is recorded in +[v1.13.11-phase16-json-integer-fidelity.md](v1.13.11-phase16-json-integer-fidelity.md). + +Phase 17 inventories all 70 non-DDL production mutations and hardens the 20 +required-row gaps with a shared internal affected-row validator. Required +container, storage, recovery, repair, remove, and GC mutations now fail before +commit on missing targets; valid zero-row cleanup, recovery, CAS, upsert, bulk, +and GC behavior remains unchanged. SQLite/PostgreSQL parity and rollback are +proven without schema, migration, dependency, public API, retry-policy, +coordination, or workflow changes. The evidence is recorded in +[v1.13.11-phase17-fail-closed-sql-mutations.md](v1.13.11-phase17-fail-closed-sql-mutations.md). + +Phase 18 makes selected existing PostgreSQL mutation-cardinality, +storage/recovery, and Linux coordination execution proof fail closed in +required CI without adding jobs, topology, product, schema, migration, +dependency, or public-API changes. Its evidence is recorded in +[v1.13.11-phase18-required-backend-coordination-ci.md](v1.13.11-phase18-required-backend-coordination-ci.md). +Phase 19 reconciles the validation matrix, backend claim matrix, reusable +release checklist, active trackers, Phase 18 closure chronology, proof +boundaries, deferred items, and evidence links without changing production, +tests, CI policy, schema, migrations, dependencies, or public APIs. Its +evidence is recorded in +[v1.13.11-phase19-validation-evidence-reconciliation.md](v1.13.11-phase19-validation-evidence-reconciliation.md). +Phase 20 freezes the pre-release state and exact-head gate contract. The +immutable candidate is the commit containing that record; it does not embed +its own SHA. One pull request to `main` is authorized only after that candidate +passes candidate-head CI, Required Gate, CodeQL, and the complete clean local +Profile A gate. Runtime results remain external evidence so successful +validation cannot invalidate its own candidate. Merge, tag, publication, and +release-branch deletion remain later operations. + +The earlier backend, selector, mutation, transaction, and benchmark-governance +evidence remains authoritative within its documented boundaries. Hosted +benchmark integrity remains hard-required, hosted timing remains advisory, and +hard timing-regression enforcement remains deferred to controlled +infrastructure. + +## Included + +- Backend compatibility claim classification and executable proof. +- SQLite/PostgreSQL shared fixtures and contract tests. +- CI activation of currently skipped PostgreSQL package tests. +- Selected engine read and mutation parity, including explicit + unsupported-backend behavior. +- Snapshot selector preservation and determinism. +- Conservative same-host repository process locking, startup-recovery + coordination, live-GC barrier behavior, and multi-process contention proof. +- Container range and header/catalog consistency, bounded decompression, exact + JSON integer preservation, and targeted SQL affected-row/iterator handling. +- CI and release-evidence updates. + +## Excluded + +- Deferred catalog-method implementation; complete CLI thin-wrapper migration; + broad engine/catalog architecture migration; recursive `Engine.Store`; and + active Repair or Recover Engine methods. +- SQLite-default initialization or embedded portable SQLite catalog product + behavior. +- Daemon, queue, API, UI, scheduling, NAS, cloud, distributed, or multi-user + work; cross-machine locking; repository redesign; and storage-format redesign. + +## Invariants + +PostgreSQL compatibility remains required. The default runtime backend remains +unchanged. SQLite-first is a future local-product direction, not a v1.13.11 +implementation. v2.0 work has not started. diff --git a/docs/release/v1.13/v1.13.11-validation-checklist.md b/docs/release/v1.13/v1.13.11-validation-checklist.md new file mode 100644 index 00000000..f5bfc3d8 --- /dev/null +++ b/docs/release/v1.13/v1.13.11-validation-checklist.md @@ -0,0 +1,348 @@ +# Coldkeep v1.13.11 Validation Checklist + +**Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` +**Status:** Ready for release +**Branch:** `release/v1.13.11` + +## Global controls + +- [x] All phases are on `release/v1.13.11`. +- [x] One final pull request only; no premature PR. +- [x] One phase-scoped commit at a time. +- [x] No version change before Phase 1. +- [x] No SQLite-default switch or v2.x product work. +- [x] No schema or storage-format change unless a later phase explicitly + authorizes one. +- [x] Exact-head validation is required before PR authorization. +- [x] Local, CI, merge, tag, and publication evidence must remain distinct. + +## Phase checks + +## Phase 0 — Complete + +**Phase status:** Complete + +- [x] v1.13.10 is recorded as a valid released baseline, not a failed release. +- [x] Its former final-v1.x conclusion is marked superseded without rewriting + historical evidence. +- [x] v1.13.11–v1.13.13 are restored in the authoritative release train. +- [x] Canonical v1.13.11 trackers exist and only authorized documentation changed. + +## Phase 1 — Complete + +**Phase status:** Complete + +- [x] Source version is `1.13.11`. +- [x] Version test expects `1.13.11`. +- [x] Reusable pre-release checklist expects `1.13.11`. +- [x] CLI human version reports `1.13.11`. +- [x] CLI JSON version reports `1.13.11`. +- [x] Changelog has a top unreleased v1.13.11 entry. +- [x] Active tracker identity is consistent. +- [x] Release-state validator passes with zero violations. +- [x] No unauthorized runtime or architecture work occurred. + +## Phase 2 — Complete + +**Phase status:** Complete + +- [x] Backend Compatibility Claim Matrix is complete. +- [x] Current SQLite/PostgreSQL claims are classified from source, tests, and CI. +- [x] Required-CI execution versus skipped PostgreSQL-gated package evidence is recorded. +- [x] Intentional backend differences, deferred APIs, evidence gaps, and owners are recorded. +- [x] No runtime, schema, CI, harness, or locking implementation work occurred. + +## Phase 3 — Complete + +**Phase status:** Complete + +- [x] Reusable Dual-Backend Test Harness is complete. +- [x] File-backed SQLite fixtures apply canonical session pragmas and current-schema setup. +- [x] PostgreSQL scratch lifecycle reports setup and cleanup failures instead of hiding them. +- [x] Existing catalog contract tests use the harness without changing their assertions. +- [x] No CI activation, production backend behavior, schema, or locking implementation occurred. + +## Phase 4 — Complete + +**Phase status:** Complete + +- [x] Existing PostgreSQL-Gated Package Suite CI Activation is implemented in `correctness-matrix` plain codec CI. +- [x] JSON proof requires PostgreSQL pass events from the harness, catalog, DB, engine, and maintenance packages. +- [x] CI audit enforces the blocking package-contract step and required environment. +- [x] Remote CI run `29729981751` proves the expected PostgreSQL JSON pass events. + +## Phase 5 — Complete + +**Phase status:** Complete + +- [x] Outcome-oriented SQLite/PostgreSQL schema contract tests and legacy-fixture inventory are implemented. +- [x] Existing required package-contract CI JSON proof explicitly selects Phase 5 PostgreSQL subtests. +- [x] Local SQLite schema-package test run completed. +- [x] Required CI proves PostgreSQL `/postgres` SCH pass events. +- [x] Canonical lint and vet succeeded. +- [x] Deterministic G6 shared packed-block corruption reproduction completed. +- [x] Fail-closed shared packed-block correction preserves single-member cleanup. +- [x] Targeted plain and AES-GCM G6 validation completed. +- [x] Exact-head adversarial validation succeeded. +- [x] The final authorized benchmark retry succeeded. +- [x] Final CI Required Gate succeeded. + +## Phase 6 — Complete + +**Phase status:** Complete + +- [x] CAT-001–007 shared catalog contract coverage is implemented. +- [x] SQLite catalog contract and race suites pass locally. +- [x] Required CI run `29983479388` proves all six strengthened catalog `/postgres` contract events at `db12c3d2`. +- [x] CAT-004 deterministic equal-time ordering correction is exact-head verified. +- [x] Implemented Catalog Contract Parity is complete. + +## Phase 7 — Complete + +**Phase status:** Complete + +- [x] ENG-R-001–004 shared-fixture contracts and the pre-cancelled `Verify` + regression are implemented and pass locally on SQLite. +- [x] The existing plain correctness-matrix JSON parser and local CI audit + require all four Phase 7 PostgreSQL `/postgres` events. +- [x] Required CI run `29993172886` records all four PostgreSQL engine + contract pass events at corrective commit `313d0069`. +- [x] Quality proves the SQLite package path after the bounded deep-verification + single-connection regression correction. +- [x] The exact-head aggregate `CI Required Gate` job `89165096957` succeeds. +- [x] Engine Read-Side Backend Parity is complete. + +## Phase 8 — Complete + +**Phase status:** Complete + +- [x] Snapshot Selector Determinism Closure is complete. +- [x] SEL-001–005 SQLite selector contracts and invalid direct-engine regex + rejection are implemented. +- [x] Required plain-codec CI JSON selectors and CI-audit enforcement name both + Phase 8 PostgreSQL subtests. +- [x] Exact-head CI run `30109561344` records both Phase 8 `/postgres` pass + events in plain correctness job `89535269535`; quality job `89535269631` and + aggregate required-gate job `89540053230` also passed. +- [x] BKC-011 classification is reconciled only after exact-head CI proof. + +## Phase 9 — Complete + +**Phase status:** Complete + +Implementation `848e579b` passed exact-head CI run `30114444798`. + +- [x] MUT-001–008 are represented by five shared backend contracts and semantic + repository/container/destination fingerprints. +- [x] Focused SQLite contracts pass once, ten repeated runs, and the race + profile. +- [x] Full nearby packages, focused restore race suites, vet, canonical lint, + release-state checks, repository guards, local CI enforcement, and diff + hygiene pass. +- [x] The existing plain-codec CI JSON parser and CI-enforcement audit require + all five Phase 9 PostgreSQL `/postgres` events. +- [x] Exact-head required CI records all five Phase 9 PostgreSQL pass events, + quality, and aggregate required-gate job `89555865026` green. +- [x] BKC-012 and BKC-013 are reconciled within their scoped Phase 9 proof. +- [x] Engine Mutation Backend Parity is complete. + +## Phase 10 — Complete + +**Phase status:** Complete + +- [x] TXN-001–009 are represented by four shared DB contracts and one + production container-helper integration contract. +- [x] Focused SQLite contracts pass once, five repeated runs, and the race + profile. +- [x] PostgreSQL blocking tests use distinct physical connections, exact + backend-PID lock observation, bounded contexts, buffered completion, and + deterministic cleanup. +- [x] The existing plain-codec CI package invocation includes + `./internal/container` once and requires all five Phase 10 PostgreSQL events. +- [x] Full nearby packages, vet, canonical lint, release-state checks, + repository guards, local CI enforcement, and diff hygiene pass. +- [x] Exact-head run `30148670910` records all five PostgreSQL pass events in + plain job `89655223183`, quality, and required gate `89656972706` green. +- [x] BKC-003 and BKC-015 are reconciled only within the Phase 10 backend- + specific proof boundary. +- [x] Backend Transaction and Row-Lock Semantics is complete. + +## Phase 11 — Complete + +**Phase status:** Complete + +- [x] Exclusive-only identity, operation policy, stable errors, diagnostic + owner metadata, and lease lifecycle are implemented without an OS lock. +- [x] Fake-based identity, policy, owner, lifecycle, cancellation, release, and + nested-acquisition contract tests are implemented. +- [x] No workflow selector, schema, Engine behavior, native lock, + subprocess proof, advisory-lock change, or SQLite live-GC change is included. +- [x] Focused, race, package, cross-platform compile, static, and repository + validation is recorded in the Phase 11 implementation artifact. +- [x] Binary-identical bounded-v2 run `30696834430` completed all four profiles + with valid functional evidence and rejected hosted paired timing authority. +- [x] Required CI is split into hard four-profile `benchmark-integrity` and + operationally required four-profile `benchmark-timing-advisory` families. +- [x] Advisory exit/report verification preserves historical threshold + crossings without downgrading evidence, comparator, or artifact failures. +- [x] Historical v1.9 files remain byte-for-byte frozen and are cited only as + `historical_v1.9_absolute` advisory inputs. +- [x] Paired production remains hard-disabled; no paired required job, + reference manifest, or numeric paired threshold policy exists. +- [x] Exact-head local validation, four integrity passes, successful advisory + evaluation, complete artifacts, and green CI/CodeQL are recorded. +- [x] Controlled-infrastructure performance enforcement remains explicitly + deferred under a separately authorized design. +- [x] Exact-head CI closure is recorded. +- [x] Repository Coordination Contract is complete. + +## Phase 12 — Complete + +**Phase status:** Complete + +- [x] `.coldkeep-control` preparation preserves identity and rejects unsafe + control paths. +- [x] Diagnostic owner metadata and the process-global non-reentrant registry + are implemented with bounded and stale-token-safe behavior. +- [x] Linux/Darwin `flock` and Windows `LockFileEx` backends provide + persistent, fail-fast, exclusive native ownership. +- [x] The production Coordinator and CLI acquire before recovery and database + work and release after operation/runtime cleanup. +- [x] Stable coordination errors, direct-caller responsibilities, and + Restore/Verify/GC outer-Lease behavior are implemented and tested. +- [x] Reconciliation run `31258313120` proves native runtime and production + Coordinator lifecycle on Linux, macOS, and Windows with a green Required + Gate and no selected native test skips. +- [x] Phase 13A reconciles the historical G6 overlap assumption without a + production coordination change and proves Linux independent-process + fail-fast contention. +- [x] Phase 12 closure evidence is recorded in + `v1.13.11-phase12-closure.md`. +- [x] Cross-Platform Repository Lock Implementation is complete. + +## Phase 13 — Complete + +**Phase status:** Complete + +- [x] Phase 13A independent-process G6 reconciliation is complete. +- [x] Phase 13B killed-process release and reacquisition proof is complete. +- [x] Live-GC cross-process and dedicated PostgreSQL advisory-session proof is + complete. +- [x] Phase 13 closure evidence is recorded in + `v1.13.11-phase13-closure.md`. +- [x] Multi-Process Contention and Live-GC Barrier Proof is complete. + +## Phase 14 — Complete + +**Phase status:** Complete + +- [x] Container Range and Header Consistency Hardening is complete. +- [x] Invalid outer ranges fail before allocation or filesystem reads while + zero-length, byte-64, exact-EOF, and exact-limit boundaries remain valid. +- [x] Supported v0/v1 header maxima are structurally valid and agree with the + catalog maximum and physical-size bound. +- [x] Packed readers use persisted `container.max_size`, and header short writes + and append overflow fail closed through established sentinels. +- [x] Phase 14 evidence is recorded in + `v1.13.11-phase14-container-range-header-consistency.md`. + +## Phase 15 — Complete + +**Phase status:** Complete + +- [x] Bounded Decompression is complete. +- [x] Identity and zstd reject negative and over-maximum expectations before + decoder creation or output allocation and require exact final size. +- [x] Zstd output and decoder memory/window resources are independently bounded + by the fixed 4 MiB format/runtime invariant. +- [x] Short, long, zero, malformed, truncated, concatenated-frame, AES-GCM, + Restore cleanup, Verify classification, and semantic-reuse cases are proven. +- [x] Released packed-writer output is arithmetically proven below the reader + ceiling, and representative legacy/current compatibility tests pass. +- [x] Phase 15 evidence is recorded in + `v1.13.11-phase15-bounded-decompression.md`. + +## Phase 16 — Complete + +**Phase status:** Complete + +- [x] JSON Integer Fidelity is complete. +- [x] The stats, inspect, and simulate-GC generic envelope path uses recursive + `json.Number` decoding and requires EOF after one internally marshaled value. +- [x] `2^53-1`, `2^53`, `2^53+1`, negative `2^53+1`, and `MaxInt64` exact + token behavior is proven without a test-side `float64` conversion. +- [x] Nested objects, maps, arrays, ordinary integers, legitimate floats, null, + booleans, strings, envelope structure, ordering, escaping, and newline + behavior remain covered. +- [x] Error JSON, exit behavior, stdout/stderr separation, coordinated spool + replay, schemas, storage formats, public APIs, dependencies, and workflows + remain unchanged. +- [x] Focused, package, full, race, static, repository, and cross-platform + validation passed, followed by green implementation-head CI and CodeQL. +- [x] Phase 16 evidence is recorded in + `v1.13.11-phase16-json-integer-fidelity.md`. + +## Phase 17 — Complete + +**Phase status:** Complete + +- [x] Fail-Closed SQL Mutation Audit is complete. +- [x] All 70 non-DDL production mutations are inventoried and classified. +- [x] M17-001 through M17-020 fail closed on required affected-row mismatch, + including exact-N repair and `RowsAffected` error handling. +- [x] Zero-safe cleanup, recovery, CAS, upsert, bulk, rebuild, and GC paths are + preserved. +- [x] SQLite/PostgreSQL parity, transaction rollback, and + no-physical-delete-on-GC-mismatch behavior are proven. +- [x] Focused, PostgreSQL, SQLite, full, race, static, cross-platform, + repository, implementation-head CI, and CodeQL validation passed. +- [x] Phase 17 evidence is recorded in + `v1.13.11-phase17-fail-closed-sql-mutations.md`. + +## Phase 18 — Complete + +**Phase status:** Complete + +- [x] Required PostgreSQL mutation-cardinality, storage/recovery, and Linux + independent-process, killed-holder, and live-GC JSON pass events are + fail-closed in existing required CI jobs. +- [x] SQLite broad quality proof, native cross-platform Coordinator proof, + advisory-session proof, required-gate topology, benchmark governance, and + CodeQL separation remain intact. +- [x] Final implementation head `eaa5896` passed CI run `31872189672`, Required + Gate job `94984536507`, CodeQL run `31872189661`, and the Phase 18 local + validation and read-only audit. +- [x] Closure head `9c1fa524` passed CI run `31873436272`, Required Gate job + `94987546529`, CodeQL run `31873436304`, and CodeQL Aggregate job + `94985540614`. +- [x] Phase 18 evidence is recorded in + `v1.13.11-phase18-required-backend-coordination-ci.md`. + +## Phase 19 — Complete + +**Phase status:** Complete + +- [x] The complete validation matrix is reconciled through Phases 14–18 and + distinguishes automated coverage from named required hosted proof. +- [x] G6, backend coordination, container, decompression, JSON, SQL mutation, + benchmark, and CI-required proof boundaries are truthful and bounded. +- [x] BKC-016 remains `Deferred — documented`; all other deferred and + unsupported items are classified without reopening historical evidence. +- [x] Active release trackers, the release train, Phase 18 closure chronology, + evidence links, and Phase 20 prerequisites are consistent. +- [x] Validation Matrix and Release Evidence Reconciliation is complete. + +## Phase 20 — Complete + +**Phase status:** Complete + +- [x] The pre-release trackers and canonical gate contract define one + immutable exact-head candidate without embedding a self-referential SHA. +- [x] Phase 20 requires candidate-head CI, Required Gate, and CodeQL plus the + complete clean local Profile A gate before one pull request is authorized. +- [x] Runtime evidence remains external, and any post-gate tracked edit + creates a new candidate requiring complete hosted and local revalidation. +- [x] BKC-016 remains `Deferred — documented`; benchmark integrity remains + hard-required and timing remains advisory. +- [x] Merge, tag, publication, and release-branch deletion remain separate and + unauthorized. diff --git a/docs/release/v1.13/v1.13.x-release-train.md b/docs/release/v1.13/v1.13.x-release-train.md index 6aa1223b..b461f336 100644 --- a/docs/release/v1.13/v1.13.x-release-train.md +++ b/docs/release/v1.13/v1.13.x-release-train.md @@ -581,23 +581,64 @@ CI `29634833195`, main CodeQL `29634833186`, and release/tag CI `29635527822`. The release branch was deleted, no Phase 25 was required, and the final verdict was `READY WITH NON-BLOCKING DEFERRALS`. +## Current authoritative release train (post-release correction) + +The earlier retirement of v1.13.11–v1.13.13 is retained below as historical +planning evidence. It was superseded after the valid v1.13.10 release by a +roadmap-to-code audit that found remaining must-before-v2 commitments. This is +the one authoritative current release train; v2.0 implementation has not +started. + ### `v1.13.10 — v1.x Closure Integrity and CI Runtime Hygiene` -**Status:** Ready for release - -This is the single current v1.13.10 definition. It closes release-documentation -integrity, reconciles the release train, makes engine-contract documentation -truthful, adds release-state validator work, maintains GitHub Actions runtime -hygiene, and freezes the v1.x/v2.0 handoff. It does not add runtime features, -change formats or schemas, or change backend defaults. - -The earlier proposed v1.13.10 through v1.13.13 objectives below are preserved -as historical planning evidence. Their completed decisions, inventories, and -readiness work, along with their deferred substantive work, are reconciled in -[`v1.13.10-release-train-reconciliation.md`](v1.13.10-release-train-reconciliation.md). -They are not separate active releases. All v1.13.10 phases are complete, its -local pre-release gate is passed awaiting publication, and one pull request is -authorized; merge, tag, publication, and external CI remain later evidence. +**Status:** Released and operationally closed + +v1.13.10 is an immutable valid released baseline for closure integrity, +release-state validation, CI runtime hygiene, and truthful known-limitations +documentation. Local evidence confirms annotated tag `v1.13.10` peels to +`423c57815580c39bee4f79ecd81570e9cfa9d273`, the 2026-07-19 merge commit for +PR #105. Its former final-v1.x conclusion was based on a narrower completion +definition and is superseded, not erased. Public GitHub evidence confirms the +stable release was published July 19, 2026 at 18:01; tag CI run #502 succeeded +with 19 jobs in 18m26s; and `release/v1.13.10` is absent from the public branch +list. The local GitHub CLI token was invalid; public pages supplied this +independent evidence. + +### `v1.13.11 — Safety and Backend Compatibility Gate Closure` + +**Status:** Ready for release; Phases 0–20 complete + +Primary scope: executable SQLite/PostgreSQL compatibility proof; same-host +repository coordination; live-GC barrier proof; deterministic snapshot +selection; bounded container reads and decompression; metadata consistency; +JSON integer fidelity; targeted fail-closed SQL mutations; and required CI +evidence. Canonical trackers are the v1.13.11 Phase 0 baseline, scope, phase +list, validation checklist, and release gate. Phase 19 reconciled the aggregate +validation/evidence contracts and preserved their bounded proof claims without +product or CI-policy changes. Phase 20 freezes the prospective exact-head gate +contract and pre-release state. The immutable candidate must pass +candidate-head CI, Required Gate, CodeQL, and the complete clean local Profile +A gate before one pull request to `main` is authorized; merge, tag, and +publication remain later operations. + +### `v1.13.12 — Engine and Catalog Completion` + +**Status:** Planned after v1.13.11 + +Implement deferred catalog methods, complete engine-owned orchestration, remove +remaining user-facing direct DB and lower-layer CLI paths, complete thin-wrapper +proof, stabilize engine-neutral contracts, and resolve Repair and Recover via +an explicit headless maintenance boundary. + +### `v1.13.13 — Final v1.x and v2 Handoff Gate` + +**Status:** Planned after v1.13.12 + +Audit the expanded roadmap against runtime code; rerun correctness, race, +adversarial, cross-platform, dual-backend, long-run, smoke, and benchmark +evidence; require no unexplained partial routing or promised-but-unimplemented +v1 surfaces; freeze one authoritative roadmap; and declare v1.x complete only +after executable evidence. ## Historical proposed continuation and final disposition diff --git a/go.mod b/go.mod index 4062541b..be8df6e3 100644 --- a/go.mod +++ b/go.mod @@ -9,3 +9,5 @@ require github.com/lib/pq v1.11.2 require github.com/mattn/go-sqlite3 v1.14.24 require github.com/klauspost/compress v1.18.0 + +require golang.org/x/sys v0.38.0 diff --git a/go.sum b/go.sum index 8aa274ad..2596ac57 100644 --- a/go.sum +++ b/go.sum @@ -4,3 +4,5 @@ github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/internal/benchmark/gate_profile.go b/internal/benchmark/gate_profile.go new file mode 100644 index 00000000..ced781a7 --- /dev/null +++ b/internal/benchmark/gate_profile.go @@ -0,0 +1,155 @@ +package benchmark + +const ( + // CIStableV1FixtureID identifies the fixed Phase 11 calibration candidate. + // Changing any value in this profile requires a new fixture identifier. + CIStableV1FixtureID = "ci-stable-v1" + // Paired fixture IDs identify immutable profile-specific inputs used by the + // paired Phase 11 benchmark contract. Historical IDs remain supported. + CIPairedW1V1FixtureID = "ci-paired-w1-v1" + CIPairedW4V1FixtureID = "ci-paired-w4-v1" + CIPairedW1V2FixtureID = "ci-paired-w1-v2" + CIPairedW4V2FixtureID = "ci-paired-w4-v2" + FixtureSeed = int64(1701) +) + +// FixtureCase identifies one ordered benchmark case and its deterministic seed. +type FixtureCase struct { + Name string `json:"name"` + Seed int64 `json:"seed"` +} + +// FixtureDescriptor records the inputs that define a benchmark fixture. +type FixtureDescriptor struct { + ID string `json:"id"` + Seed int64 `json:"seed"` + LargeFileSizeBytes int64 `json:"large_file_size_bytes"` + ManySmallFileCount int `json:"many_small_file_count"` + ManySmallFileSizeBytes int `json:"many_small_file_size_bytes"` + MixedFileCount int `json:"mixed_file_count"` + MixedMinFileSizeBytes int `json:"mixed_min_file_size_bytes"` + MixedMaxFileSizeBytes int `json:"mixed_max_file_size_bytes"` + RemoveEvery int `json:"remove_every"` + CaseDatabaseIsolation bool `json:"case_database_isolation"` + OrderedCases []FixtureCase `json:"ordered_cases"` +} + +// CIStableV1ScenarioConfig is the fixed release-gate calibration candidate. +func CIStableV1ScenarioConfig() ScenarioConfig { + return ScenarioConfig{ + Seed: FixtureSeed, + LargeFileSizeBytes: 96 * 1024 * 1024, + ManySmallFileCount: 600, + ManySmallFileSizeBytes: 1024, + MixedFileCount: 400, + MixedMinFileSizeBytes: 1024, + MixedMaxFileSizeBytes: 256 * 1024, + RemoveEvery: 4, + CaseDatabaseIsolation: true, + } +} + +// CIPairedW1V1ScenarioConfig is the fixed workers=1 paired-gate fixture. +func CIPairedW1V1ScenarioConfig() ScenarioConfig { + return ScenarioConfig{ + Seed: FixtureSeed, + LargeFileSizeBytes: 96 * 1024 * 1024, + ManySmallFileCount: 600, + ManySmallFileSizeBytes: 1024, + MixedFileCount: 400, + MixedMinFileSizeBytes: 1024, + MixedMaxFileSizeBytes: 256 * 1024, + RemoveEvery: 4, + CaseDatabaseIsolation: true, + } +} + +// CIPairedW4V1ScenarioConfig is the fixed workers=4 paired-gate fixture. +func CIPairedW4V1ScenarioConfig() ScenarioConfig { + return ScenarioConfig{ + Seed: FixtureSeed, + LargeFileSizeBytes: 128 * 1024 * 1024, + ManySmallFileCount: 1200, + ManySmallFileSizeBytes: 1024, + MixedFileCount: 800, + MixedMinFileSizeBytes: 1024, + MixedMaxFileSizeBytes: 256 * 1024, + RemoveEvery: 4, + CaseDatabaseIsolation: true, + } +} + +// CIPairedW1V2ScenarioConfig is the bounded workers=1 paired diagnostic fixture. +func CIPairedW1V2ScenarioConfig() ScenarioConfig { + return ScenarioConfig{ + Seed: FixtureSeed, + LargeFileSizeBytes: 64 * 1024 * 1024, + ManySmallFileCount: 400, + ManySmallFileSizeBytes: 1024, + MixedFileCount: 400, + MixedMinFileSizeBytes: 1024, + MixedMaxFileSizeBytes: 256 * 1024, + RemoveEvery: 4, + CaseDatabaseIsolation: true, + } +} + +// CIPairedW4V2ScenarioConfig is the bounded workers=4 paired diagnostic fixture. +func CIPairedW4V2ScenarioConfig() ScenarioConfig { + return ScenarioConfig{ + Seed: FixtureSeed, + LargeFileSizeBytes: 64 * 1024 * 1024, + ManySmallFileCount: 400, + ManySmallFileSizeBytes: 1024, + MixedFileCount: 800, + MixedMinFileSizeBytes: 1024, + MixedMaxFileSizeBytes: 256 * 1024, + RemoveEvery: 4, + CaseDatabaseIsolation: true, + } +} + +// RequiresCaseDatabaseIsolation reports whether a preset owns a fresh database +// and filesystem environment for every benchmark case. +func RequiresCaseDatabaseIsolation(preset DatasetPreset) bool { + switch preset { + case DatasetPresetCIStableV1, + DatasetPresetCIPairedW1V1, DatasetPresetCIPairedW4V1, + DatasetPresetCIPairedW1V2, DatasetPresetCIPairedW4V2: + return true + default: + return false + } +} + +// FixtureDescriptorFor returns the deterministic descriptor for a preset. +func FixtureDescriptorFor(preset DatasetPreset, cfg ScenarioConfig) FixtureDescriptor { + cfg = cfg.withDefaults() + id := string(preset) + if preset == DatasetPresetCIStableV1 { + id = CIStableV1FixtureID + } + return FixtureDescriptor{ + ID: id, + Seed: cfg.Seed, + LargeFileSizeBytes: cfg.LargeFileSizeBytes, + ManySmallFileCount: cfg.ManySmallFileCount, + ManySmallFileSizeBytes: cfg.ManySmallFileSizeBytes, + MixedFileCount: cfg.MixedFileCount, + MixedMinFileSizeBytes: cfg.MixedMinFileSizeBytes, + MixedMaxFileSizeBytes: cfg.MixedMaxFileSizeBytes, + RemoveEvery: cfg.RemoveEvery, + CaseDatabaseIsolation: cfg.CaseDatabaseIsolation, + OrderedCases: []FixtureCase{ + {Name: "store-large-file", Seed: cfg.Seed + 11}, + {Name: "store-many-small-files", Seed: cfg.Seed + 21}, + {Name: "store-mixed-dataset", Seed: cfg.Seed + 31}, + {Name: "restore-large-file", Seed: cfg.Seed + 41}, + {Name: "restore-many-files", Seed: cfg.Seed + 51}, + {Name: "snapshot-creation", Seed: cfg.Seed + 61}, + {Name: "gc-after-churn", Seed: cfg.Seed + 71}, + {Name: "stats-inspect", Seed: cfg.Seed + 81}, + {Name: "verify-system-deep", Seed: cfg.Seed + 91}, + }, + } +} diff --git a/internal/benchmark/gate_profile_test.go b/internal/benchmark/gate_profile_test.go new file mode 100644 index 00000000..abc3e9d2 --- /dev/null +++ b/internal/benchmark/gate_profile_test.go @@ -0,0 +1,144 @@ +package benchmark + +import "testing" + +func TestCIStableV1FixtureContract(t *testing.T) { + cfg := CIStableV1ScenarioConfig() + if cfg.Seed != 1701 || + cfg.LargeFileSizeBytes != 96*1024*1024 || + cfg.ManySmallFileCount != 600 || + cfg.ManySmallFileSizeBytes != 1024 || + cfg.MixedFileCount != 400 || + cfg.MixedMinFileSizeBytes != 1024 || + cfg.MixedMaxFileSizeBytes != 256*1024 || + cfg.RemoveEvery != 4 || + !cfg.CaseDatabaseIsolation { + t.Fatalf("unexpected ci-stable-v1 config: %+v", cfg) + } + + descriptor := FixtureDescriptorFor(DatasetPresetCIStableV1, cfg) + if descriptor.ID != CIStableV1FixtureID || !descriptor.CaseDatabaseIsolation { + t.Fatalf("unexpected descriptor: %+v", descriptor) + } + wantCases := []string{ + "store-large-file", + "store-many-small-files", + "store-mixed-dataset", + "restore-large-file", + "restore-many-files", + "snapshot-creation", + "gc-after-churn", + "stats-inspect", + "verify-system-deep", + } + if len(descriptor.OrderedCases) != len(wantCases) { + t.Fatalf("case count: got=%d want=%d", len(descriptor.OrderedCases), len(wantCases)) + } + for index, want := range wantCases { + got := descriptor.OrderedCases[index] + if got.Name != want { + t.Fatalf("case %d: got=%q want=%q", index, got.Name, want) + } + wantSeed := FixtureSeed + int64((index+1)*10+1) + if got.Seed != wantSeed { + t.Fatalf("seed for %s: got=%d want=%d", got.Name, got.Seed, wantSeed) + } + } +} + +func TestCIPairedFixtureContracts(t *testing.T) { + tests := []struct { + name string + preset DatasetPreset + fixtureID string + config ScenarioConfig + largeBytes int64 + manyCount int + mixedCount int + }{ + { + name: "workers-1", + preset: DatasetPresetCIPairedW1V1, + fixtureID: CIPairedW1V1FixtureID, + config: CIPairedW1V1ScenarioConfig(), + largeBytes: 96 * 1024 * 1024, + manyCount: 600, + mixedCount: 400, + }, + { + name: "workers-4", + preset: DatasetPresetCIPairedW4V1, + fixtureID: CIPairedW4V1FixtureID, + config: CIPairedW4V1ScenarioConfig(), + largeBytes: 128 * 1024 * 1024, + manyCount: 1200, + mixedCount: 800, + }, + { + name: "workers-1-v2", + preset: DatasetPresetCIPairedW1V2, + fixtureID: CIPairedW1V2FixtureID, + config: CIPairedW1V2ScenarioConfig(), + largeBytes: 64 * 1024 * 1024, + manyCount: 400, + mixedCount: 400, + }, + { + name: "workers-4-v2", + preset: DatasetPresetCIPairedW4V2, + fixtureID: CIPairedW4V2FixtureID, + config: CIPairedW4V2ScenarioConfig(), + largeBytes: 64 * 1024 * 1024, + manyCount: 400, + mixedCount: 800, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := test.config + if cfg.Seed != FixtureSeed || + cfg.LargeFileSizeBytes != test.largeBytes || + cfg.ManySmallFileCount != test.manyCount || + cfg.ManySmallFileSizeBytes != 1024 || + cfg.MixedFileCount != test.mixedCount || + cfg.MixedMinFileSizeBytes != 1024 || + cfg.MixedMaxFileSizeBytes != 256*1024 || + cfg.RemoveEvery != 4 || + !cfg.CaseDatabaseIsolation { + t.Fatalf("unexpected paired config: %+v", cfg) + } + descriptor := FixtureDescriptorFor(test.preset, cfg) + if descriptor.ID != test.fixtureID { + t.Fatalf("unexpected descriptor: %+v", descriptor) + } + wantCases := []string{ + "store-large-file", + "store-many-small-files", + "store-mixed-dataset", + "restore-large-file", + "restore-many-files", + "snapshot-creation", + "gc-after-churn", + "stats-inspect", + "verify-system-deep", + } + if len(descriptor.OrderedCases) != len(wantCases) { + t.Fatalf("case count: got=%d want=%d", len(descriptor.OrderedCases), len(wantCases)) + } + for index, want := range wantCases { + got := descriptor.OrderedCases[index] + if got.Name != want { + t.Fatalf("case %d: got=%q want=%q", index, got.Name, want) + } + wantSeed := FixtureSeed + int64((index+1)*10+1) + if got.Seed != wantSeed { + t.Fatalf("seed for %s: got=%d want=%d", got.Name, got.Seed, wantSeed) + } + } + if !RequiresCaseDatabaseIsolation(test.preset) { + t.Fatalf("paired preset must require isolation: %q", test.preset) + } + }) + } +} diff --git a/internal/benchmark/report.go b/internal/benchmark/report.go index 6c88d226..61bc730d 100644 --- a/internal/benchmark/report.go +++ b/internal/benchmark/report.go @@ -9,9 +9,14 @@ import ( type DatasetPreset string const ( - DatasetPresetSmall DatasetPreset = "small" - DatasetPresetMedium DatasetPreset = "medium" - DatasetPresetLarge DatasetPreset = "large" + DatasetPresetSmall DatasetPreset = "small" + DatasetPresetMedium DatasetPreset = "medium" + DatasetPresetLarge DatasetPreset = "large" + DatasetPresetCIStableV1 DatasetPreset = CIStableV1FixtureID + DatasetPresetCIPairedW1V1 DatasetPreset = CIPairedW1V1FixtureID + DatasetPresetCIPairedW4V1 DatasetPreset = CIPairedW4V1FixtureID + DatasetPresetCIPairedW1V2 DatasetPreset = CIPairedW1V2FixtureID + DatasetPresetCIPairedW4V2 DatasetPreset = CIPairedW4V2FixtureID ) type IterationReport struct { @@ -23,6 +28,7 @@ type RunReport struct { GeneratedAtUTC string `json:"generated_at_utc"` Dataset DatasetPreset `json:"dataset"` Repeat int `json:"repeat"` + Fixture FixtureDescriptor `json:"fixture"` Iterations []IterationReport `json:"iterations"` } @@ -33,10 +39,12 @@ func ParseDatasetPreset(raw string) (DatasetPreset, error) { } switch DatasetPreset(normalized) { - case DatasetPresetSmall, DatasetPresetMedium, DatasetPresetLarge: + case DatasetPresetSmall, DatasetPresetMedium, DatasetPresetLarge, DatasetPresetCIStableV1, + DatasetPresetCIPairedW1V1, DatasetPresetCIPairedW4V1, + DatasetPresetCIPairedW1V2, DatasetPresetCIPairedW4V2: return DatasetPreset(normalized), nil default: - return "", fmt.Errorf("invalid dataset preset %q (allowed: small, medium, large)", raw) + return "", fmt.Errorf("invalid dataset preset %q (allowed: small, medium, large, ci-stable-v1, ci-paired-w1-v1, ci-paired-w4-v1, ci-paired-w1-v2, ci-paired-w4-v2)", raw) } } @@ -72,6 +80,16 @@ func PresetScenarioConfig(preset DatasetPreset) (ScenarioConfig, error) { MixedMaxFileSizeBytes: defaultMixedMaxFileSizeBytes, RemoveEvery: defaultRemoveEvery, }, nil + case DatasetPresetCIStableV1: + return CIStableV1ScenarioConfig(), nil + case DatasetPresetCIPairedW1V1: + return CIPairedW1V1ScenarioConfig(), nil + case DatasetPresetCIPairedW4V1: + return CIPairedW4V1ScenarioConfig(), nil + case DatasetPresetCIPairedW1V2: + return CIPairedW1V2ScenarioConfig(), nil + case DatasetPresetCIPairedW4V2: + return CIPairedW4V2ScenarioConfig(), nil default: return ScenarioConfig{}, fmt.Errorf("unsupported dataset preset %q", preset) } @@ -95,18 +113,29 @@ func RunPreset(preset DatasetPreset, repeat int, base ScenarioConfig) (RunReport cfg.MixedMinFileSizeBytes = presetCfg.MixedMinFileSizeBytes cfg.MixedMaxFileSizeBytes = presetCfg.MixedMaxFileSizeBytes cfg.RemoveEvery = presetCfg.RemoveEvery + cfg.CaseDatabaseIsolation = presetCfg.CaseDatabaseIsolation + if cfg.CaseDatabaseIsolation && cfg.CaseEnvironmentFactory == nil { + return RunReport{}, fmt.Errorf("preset %q requires a per-case environment factory", preset) + } report := RunReport{ GeneratedAtUTC: time.Now().UTC().Format(time.RFC3339), Dataset: preset, Repeat: repeat, + Fixture: FixtureDescriptorFor(preset, cfg), Iterations: make([]IterationReport, 0, repeat), } for i := 1; i <= repeat; i++ { iterCfg := cfg iterCfg.RunTag = fmt.Sprintf("iter-%02d", i) - results, runErr := RunBenchmark(CoreScenarios(iterCfg)) + var results []Result + var runErr error + if iterCfg.CaseEnvironmentFactory != nil { + results, runErr = RunBenchmarkWithEnvironmentFactory(CoreScenarios(iterCfg), iterCfg.CaseEnvironmentFactory) + } else { + results, runErr = RunBenchmark(CoreScenarios(iterCfg)) + } report.Iterations = append(report.Iterations, IterationReport{ Iteration: i, Results: results, diff --git a/internal/benchmark/report_test.go b/internal/benchmark/report_test.go index 859a9ff9..43642a1a 100644 --- a/internal/benchmark/report_test.go +++ b/internal/benchmark/report_test.go @@ -1,6 +1,9 @@ package benchmark -import "testing" +import ( + "fmt" + "testing" +) func TestParseDatasetPreset(t *testing.T) { preset, err := ParseDatasetPreset("") @@ -19,6 +22,26 @@ func TestParseDatasetPreset(t *testing.T) { t.Fatalf("expected medium, got %q", preset) } + preset, err = ParseDatasetPreset("CI-STABLE-V1") + if err != nil { + t.Fatalf("ParseDatasetPreset ci-stable-v1: %v", err) + } + if preset != DatasetPresetCIStableV1 { + t.Fatalf("expected ci-stable-v1, got %q", preset) + } + + for raw, want := range map[string]DatasetPreset{ + "CI-PAIRED-W1-V1": DatasetPresetCIPairedW1V1, + "ci-paired-w4-v1": DatasetPresetCIPairedW4V1, + "CI-PAIRED-W1-V2": DatasetPresetCIPairedW1V2, + "ci-paired-w4-v2": DatasetPresetCIPairedW4V2, + } { + preset, err = ParseDatasetPreset(raw) + if err != nil || preset != want { + t.Fatalf("ParseDatasetPreset(%q): got=%q err=%v", raw, preset, err) + } + } + if _, err := ParseDatasetPreset("xlarge"); err == nil { t.Fatal("expected invalid preset error") } @@ -31,6 +54,26 @@ func TestRunPresetValidatesRepeat(t *testing.T) { } } +func TestRunPresetCIStableV1RequiresCaseEnvironmentFactory(t *testing.T) { + _, err := RunPreset(DatasetPresetCIStableV1, 1, ScenarioConfig{}) + if err == nil || err.Error() != `preset "ci-stable-v1" requires a per-case environment factory` { + t.Fatalf("expected case isolation requirement, got: %v", err) + } +} + +func TestRunPresetPairedRequiresCaseEnvironmentFactory(t *testing.T) { + for _, preset := range []DatasetPreset{ + DatasetPresetCIPairedW1V1, DatasetPresetCIPairedW4V1, + DatasetPresetCIPairedW1V2, DatasetPresetCIPairedW4V2, + } { + _, err := RunPreset(preset, 1, ScenarioConfig{}) + want := fmt.Sprintf("preset %q requires a per-case environment factory", preset) + if err == nil || err.Error() != want { + t.Fatalf("expected case isolation requirement for %q, got: %v", preset, err) + } + } +} + func TestRunPresetWithStubRunner(t *testing.T) { calls := 0 report, err := RunPreset(DatasetPresetSmall, 2, ScenarioConfig{ @@ -46,6 +89,9 @@ func TestRunPresetWithStubRunner(t *testing.T) { if report.Dataset != DatasetPresetSmall || report.Repeat != 2 { t.Fatalf("unexpected report header: %+v", report) } + if report.Fixture.ID != string(DatasetPresetSmall) { + t.Fatalf("unexpected fixture descriptor: %+v", report.Fixture) + } if len(report.Iterations) != 2 { t.Fatalf("expected 2 iterations, got %d", len(report.Iterations)) } diff --git a/internal/benchmark/runner.go b/internal/benchmark/runner.go index 58341561..e5cbf7a5 100644 --- a/internal/benchmark/runner.go +++ b/internal/benchmark/runner.go @@ -3,6 +3,7 @@ package benchmark import ( "bufio" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -23,8 +24,18 @@ type BenchmarkCase struct { type BenchmarkContext struct { RepoPath string DataPath string + ExtraEnv map[string]string } +// CaseEnvironmentFactory creates per-case environment overrides and cleanup. +// It runs before the case timer starts. +type CaseEnvironmentFactory func(caseName string) (map[string]string, func() error, error) + +// FinalStateObserver captures benchmark-only evidence after a case finishes and +// before its external resources and temporary paths are cleaned up. The raw +// message must encode one sanitized JSON object. +type FinalStateObserver func(caseName string, ctx BenchmarkContext) (json.RawMessage, error) + // Result captures one benchmark case execution outcome. type Result struct { Name string @@ -32,8 +43,11 @@ type Result struct { Metrics Metrics Execution execution.Options ExecStats execution.ExecutionStats - Success bool - Error string + // DiagnosticFinalState is separately versioned diagnostic evidence. It does + // not change the surrounding benchmark report schema. + DiagnosticFinalState json.RawMessage + Success bool + Error string } type ioDebugProcessRecord struct { @@ -48,86 +62,225 @@ type ioDebugProcessRecord struct { // RunBenchmark executes benchmark cases sequentially with isolated temp paths. func RunBenchmark(cases []BenchmarkCase) ([]Result, error) { + return runBenchmark(cases, nil, nil) +} + +// RunBenchmarkWithEnvironmentFactory executes cases with per-case external +// resources, such as isolated benchmark databases. +func RunBenchmarkWithEnvironmentFactory(cases []BenchmarkCase, factory CaseEnvironmentFactory) ([]Result, error) { + if factory == nil { + return nil, fmt.Errorf("case environment factory cannot be nil") + } + return runBenchmark(cases, factory, nil) +} + +// RunBenchmarkWithEnvironmentFactoryAndObserver executes cases with per-case +// external resources and captures final state before any cleanup runs. +func RunBenchmarkWithEnvironmentFactoryAndObserver( + cases []BenchmarkCase, + factory CaseEnvironmentFactory, + observer FinalStateObserver, +) ([]Result, error) { + if factory == nil { + return nil, fmt.Errorf("case environment factory cannot be nil") + } + if observer == nil { + return nil, fmt.Errorf("final state observer cannot be nil") + } + return runBenchmark(cases, factory, observer) +} + +func runBenchmark(cases []BenchmarkCase, factory CaseEnvironmentFactory, observer FinalStateObserver) ([]Result, error) { results := make([]Result, 0, len(cases)) for index, bc := range cases { - if strings.TrimSpace(bc.Name) == "" { - return results, fmt.Errorf("benchmark case at index %d has empty name", index) + if err := validateBenchmarkCase(index, bc); err != nil { + return results, err } - if bc.Run == nil { - return results, fmt.Errorf("benchmark case %q has nil run function", bc.Name) + result, completed, err := runBenchmarkCase(bc, factory, observer) + if completed { + results = append(results, result) } - - ctx, cleanup, err := newBenchmarkContext() if err != nil { - return results, fmt.Errorf("create context for benchmark case %q: %w", bc.Name, err) + return results, err } + } - ioCountersPath := filepath.Join(ctx.RepoPath, fmt.Sprintf(".io-debug-%s.jsonl", strings.ReplaceAll(bc.Name, " ", "_"))) - metrics, runErr := Measure(func() error { - _ = os.Remove(ioCountersPath) - - prevPath, hadPath := os.LookupEnv("COLDKEEP_IO_COUNTERS_FILE") - if err := os.Setenv("COLDKEEP_IO_COUNTERS_FILE", ioCountersPath); err != nil { - return fmt.Errorf("set io debug env: %w", err) - } - defer func() { - if hadPath { - _ = os.Setenv("COLDKEEP_IO_COUNTERS_FILE", prevPath) - } else { - _ = os.Unsetenv("COLDKEEP_IO_COUNTERS_FILE") - } - }() - - return bc.Run(ctx) - }) - - ioStats, ioErr := readAggregatedIOCounters(ioCountersPath) - if ioErr != nil { - return results, fmt.Errorf("read io counters for benchmark case %q: %w", bc.Name, ioErr) - } + return results, nil +} - cleanupErr := cleanup() - - result := Result{ - Name: bc.Name, - Duration: metrics.Duration, - Metrics: metrics, - Execution: bc.Execution, - ExecStats: execution.ExecutionStats{ - TotalFilesProcessed: metrics.FilesProcessed, - TotalBytesProcessed: metrics.BytesProcessed, - WorkersUsed: bc.Execution.StoreFolderWorkers, - ContainerAppendCount: ioStats.ContainerAppendCount, - FsyncCount: ioStats.FsyncCount, - ContainerOpenCount: ioStats.ContainerOpenCount, - ContainerCloseCount: ioStats.ContainerCloseCount, - BytesWritten: ioStats.BytesWritten, - BytesRead: ioStats.BytesRead, - SnapshotMetadataWrites: ioStats.SnapshotMetadataWrites, - }, - Success: runErr == nil && cleanupErr == nil, - } - if runErr != nil { - result.Error = runErr.Error() - } - if cleanupErr != nil { - if result.Error == "" { - result.Error = cleanupErr.Error() - } else { - result.Error = result.Error + "; " + cleanupErr.Error() - } - } - results = append(results, result) +func validateBenchmarkCase(index int, bc BenchmarkCase) error { + if strings.TrimSpace(bc.Name) == "" { + return fmt.Errorf("benchmark case at index %d has empty name", index) + } + if bc.Run == nil { + return fmt.Errorf("benchmark case %q has nil run function", bc.Name) + } + return nil +} + +func runBenchmarkCase( + bc BenchmarkCase, + factory CaseEnvironmentFactory, + observer FinalStateObserver, +) (Result, bool, error) { + ctx, cleanup, externalCleanup, err := prepareBenchmarkCase(bc.Name, factory) + if err != nil { + return Result{}, false, err + } + + ioCountersPath := benchmarkIOCountersPath(ctx, bc.Name) + metrics, runErr := measureBenchmarkCase(bc, ctx, ioCountersPath) + ioStats, ioErr := readAggregatedIOCounters(ioCountersPath) + diagnosticFinalState, observerErr := observeBenchmarkFinalState(observer, bc.Name, ctx) + cleanupErr := cleanupBenchmarkCase(externalCleanup, cleanup) + if ioErr != nil { + return Result{}, false, benchmarkIOCountersError(bc.Name, ioErr, cleanupErr) + } + + operationErr := errors.Join(runErr, observerErr) + result := benchmarkCaseResult(bc, metrics, ioStats, diagnosticFinalState, operationErr, cleanupErr) + return result, true, benchmarkCaseError(bc.Name, operationErr, cleanupErr) +} + +func prepareBenchmarkCase( + caseName string, + factory CaseEnvironmentFactory, +) (BenchmarkContext, func() error, func() error, error) { + ctx, cleanup, err := newBenchmarkContext() + if err != nil { + return BenchmarkContext{}, nil, nil, fmt.Errorf("create context for benchmark case %q: %w", caseName, err) + } + externalCleanup := func() error { return nil } + if factory == nil { + return ctx, cleanup, externalCleanup, nil + } + + extraEnv, cleanupExternal, factoryErr := factory(caseName) + if factoryErr != nil { + _ = cleanup() + return BenchmarkContext{}, nil, nil, fmt.Errorf("create environment for benchmark case %q: %w", caseName, factoryErr) + } + ctx.ExtraEnv = extraEnv + if cleanupExternal != nil { + externalCleanup = cleanupExternal + } + return ctx, cleanup, externalCleanup, nil +} + +func benchmarkIOCountersPath(ctx BenchmarkContext, caseName string) string { + return filepath.Join(ctx.RepoPath, fmt.Sprintf(".io-debug-%s.jsonl", strings.ReplaceAll(caseName, " ", "_"))) +} - if runErr != nil { - return results, fmt.Errorf("run benchmark case %q: %w", bc.Name, runErr) +func measureBenchmarkCase(bc BenchmarkCase, ctx BenchmarkContext, ioCountersPath string) (Metrics, error) { + return Measure(func() error { + _ = os.Remove(ioCountersPath) + prevPath, hadPath := os.LookupEnv("COLDKEEP_IO_COUNTERS_FILE") + if err := os.Setenv("COLDKEEP_IO_COUNTERS_FILE", ioCountersPath); err != nil { + return fmt.Errorf("set io debug env: %w", err) } - if cleanupErr != nil { - return results, fmt.Errorf("cleanup benchmark case %q context: %w", bc.Name, cleanupErr) + defer restoreBenchmarkIOCountersPath(prevPath, hadPath) + return bc.Run(ctx) + }) +} + +func restoreBenchmarkIOCountersPath(previous string, existed bool) { + if existed { + _ = os.Setenv("COLDKEEP_IO_COUNTERS_FILE", previous) + return + } + _ = os.Unsetenv("COLDKEEP_IO_COUNTERS_FILE") +} + +func observeBenchmarkFinalState( + observer FinalStateObserver, + caseName string, + ctx BenchmarkContext, +) (json.RawMessage, error) { + if observer == nil { + return nil, nil + } + diagnosticFinalState, err := observer(caseName, ctx) + if err != nil { + return diagnosticFinalState, err + } + if !json.Valid(diagnosticFinalState) { + return diagnosticFinalState, fmt.Errorf("observer returned invalid JSON") + } + var object map[string]any + if err := json.Unmarshal(diagnosticFinalState, &object); err != nil || object == nil { + return diagnosticFinalState, fmt.Errorf("observer must return a JSON object") + } + return diagnosticFinalState, nil +} + +func cleanupBenchmarkCase(externalCleanup, cleanup func() error) error { + externalCleanupErr := externalCleanup() + cleanupErr := cleanup() + if externalCleanupErr != nil && cleanupErr != nil { + return errors.Join(externalCleanupErr, cleanupErr) + } + if externalCleanupErr != nil { + return externalCleanupErr + } + return cleanupErr +} + +func benchmarkIOCountersError(caseName string, ioErr, cleanupErr error) error { + if cleanupErr != nil { + ioErr = errors.Join(ioErr, cleanupErr) + } + return fmt.Errorf("read io counters for benchmark case %q: %w", caseName, ioErr) +} + +func benchmarkCaseResult( + bc BenchmarkCase, + metrics Metrics, + ioStats ioDebugProcessRecord, + diagnosticFinalState json.RawMessage, + operationErr error, + cleanupErr error, +) Result { + result := Result{ + Name: bc.Name, + Duration: metrics.Duration, + Metrics: metrics, + Execution: bc.Execution, + ExecStats: execution.ExecutionStats{ + TotalFilesProcessed: metrics.FilesProcessed, + TotalBytesProcessed: metrics.BytesProcessed, + WorkersUsed: bc.Execution.StoreFolderWorkers, + ContainerAppendCount: ioStats.ContainerAppendCount, + FsyncCount: ioStats.FsyncCount, + ContainerOpenCount: ioStats.ContainerOpenCount, + ContainerCloseCount: ioStats.ContainerCloseCount, + BytesWritten: ioStats.BytesWritten, + BytesRead: ioStats.BytesRead, + SnapshotMetadataWrites: ioStats.SnapshotMetadataWrites, + }, + DiagnosticFinalState: diagnosticFinalState, + Success: operationErr == nil && cleanupErr == nil, + } + if operationErr != nil { + result.Error = operationErr.Error() + } + if cleanupErr != nil { + if result.Error == "" { + result.Error = cleanupErr.Error() + } else { + result.Error += "; " + cleanupErr.Error() } } + return result +} - return results, nil +func benchmarkCaseError(caseName string, operationErr, cleanupErr error) error { + if operationErr != nil { + return fmt.Errorf("run benchmark case %q: %w", caseName, operationErr) + } + if cleanupErr != nil { + return fmt.Errorf("cleanup benchmark case %q context: %w", caseName, cleanupErr) + } + return nil } func readAggregatedIOCounters(path string) (ioDebugProcessRecord, error) { diff --git a/internal/benchmark/runner_test.go b/internal/benchmark/runner_test.go index f9bfada5..29260374 100644 --- a/internal/benchmark/runner_test.go +++ b/internal/benchmark/runner_test.go @@ -1,8 +1,11 @@ package benchmark import ( + "encoding/json" + "errors" "os" "path/filepath" + "strings" "testing" "github.com/franchoy/coldkeep/internal/execution" @@ -124,3 +127,154 @@ func TestRunBenchmarkRejectsInvalidCases(t *testing.T) { t.Fatal("expected error for nil case run function") } } + +func TestRunBenchmarkWithEnvironmentFactoryScopesAndCleansEachCase(t *testing.T) { + var created []string + var cleaned []string + cases := []BenchmarkCase{ + { + Name: "first", + Execution: execution.Options{StoreFolderWorkers: 1, PipelineDepth: 1, Deterministic: true}, + Run: func(ctx BenchmarkContext) error { + if got := ctx.ExtraEnv["DB_NAME"]; got != "db_first" { + t.Fatalf("first DB_NAME=%q", got) + } + return nil + }, + }, + { + Name: "second", + Execution: execution.Options{StoreFolderWorkers: 1, PipelineDepth: 1, Deterministic: true}, + Run: func(ctx BenchmarkContext) error { + if got := ctx.ExtraEnv["DB_NAME"]; got != "db_second" { + t.Fatalf("second DB_NAME=%q", got) + } + return nil + }, + }, + } + + _, err := RunBenchmarkWithEnvironmentFactory(cases, func(caseName string) (map[string]string, func() error, error) { + created = append(created, caseName) + return map[string]string{"DB_NAME": "db_" + caseName}, func() error { + cleaned = append(cleaned, caseName) + return nil + }, nil + }) + if err != nil { + t.Fatalf("RunBenchmarkWithEnvironmentFactory: %v", err) + } + if got := len(created); got != 2 { + t.Fatalf("created=%v", created) + } + if got := len(cleaned); got != 2 { + t.Fatalf("cleaned=%v", cleaned) + } + for index := range created { + if created[index] != cleaned[index] { + t.Fatalf("cleanup order mismatch: created=%v cleaned=%v", created, cleaned) + } + } +} + +func TestRunBenchmarkWithEnvironmentFactoryRejectsNilFactory(t *testing.T) { + if _, err := RunBenchmarkWithEnvironmentFactory(nil, nil); err == nil { + t.Fatal("expected nil factory error") + } +} + +func TestRunBenchmarkWithEnvironmentFactoryCleansAfterCaseFailure(t *testing.T) { + cleaned := false + cases := []BenchmarkCase{{ + Name: "failing", + Run: func(BenchmarkContext) error { + return errors.New("case failed") + }, + }} + results, err := RunBenchmarkWithEnvironmentFactory( + cases, + func(string) (map[string]string, func() error, error) { + return map[string]string{"DB_NAME": "db_failing"}, func() error { + cleaned = true + return nil + }, nil + }, + ) + if err == nil || !strings.Contains(err.Error(), "case failed") { + t.Fatalf("expected case failure, got results=%+v err=%v", results, err) + } + if !cleaned { + t.Fatal("expected external cleanup after case failure") + } +} + +func TestRunBenchmarkObserverRunsBeforeCleanup(t *testing.T) { + var observedPath string + cleaned := false + cases := []BenchmarkCase{{ + Name: "observed", + Run: func(ctx BenchmarkContext) error { + observedPath = filepath.Join(ctx.DataPath, "final-state.txt") + return os.WriteFile(observedPath, []byte("present"), 0o600) + }, + }} + + results, err := RunBenchmarkWithEnvironmentFactoryAndObserver( + cases, + func(string) (map[string]string, func() error, error) { + return map[string]string{"DB_NAME": "ephemeral"}, func() error { + cleaned = true + return nil + }, nil + }, + func(_ string, ctx BenchmarkContext) (json.RawMessage, error) { + if cleaned { + t.Fatal("observer ran after external cleanup") + } + if _, err := os.Stat(observedPath); err != nil { + t.Fatalf("observer could not read pre-cleanup state: %v", err) + } + if _, err := os.Stat(ctx.RepoPath); err != nil { + t.Fatalf("observer could not read benchmark context: %v", err) + } + return []byte(`{"schema_version":1,"digest":"abc"}`), nil + }, + ) + if err != nil { + t.Fatalf("RunBenchmarkWithEnvironmentFactoryAndObserver: %v", err) + } + if !cleaned { + t.Fatal("external cleanup did not run") + } + if len(results) != 1 || string(results[0].DiagnosticFinalState) != `{"schema_version":1,"digest":"abc"}` { + t.Fatalf("unexpected diagnostic result: %+v", results) + } + if _, err := os.Stat(observedPath); !os.IsNotExist(err) { + t.Fatalf("benchmark context survived cleanup: %v", err) + } +} + +func TestRunBenchmarkCleanupRunsAfterObserverFailure(t *testing.T) { + cleaned := false + results, err := RunBenchmarkWithEnvironmentFactoryAndObserver( + []BenchmarkCase{{Name: "observed", Run: func(BenchmarkContext) error { return nil }}}, + func(string) (map[string]string, func() error, error) { + return nil, func() error { + cleaned = true + return nil + }, nil + }, + func(string, BenchmarkContext) (json.RawMessage, error) { + return nil, errors.New("capture failed") + }, + ) + if err == nil || !strings.Contains(err.Error(), "capture failed") { + t.Fatalf("expected observer failure, got results=%+v err=%v", results, err) + } + if !cleaned { + t.Fatal("external cleanup did not run after observer failure") + } + if len(results) != 1 || results[0].Success || !strings.Contains(results[0].Error, "capture failed") { + t.Fatalf("observer failure missing from result: %+v", results) + } +} diff --git a/internal/benchmark/scenarios.go b/internal/benchmark/scenarios.go index 7023a80c..59561744 100644 --- a/internal/benchmark/scenarios.go +++ b/internal/benchmark/scenarios.go @@ -51,6 +51,8 @@ type ScenarioConfig struct { RunTag string ExtraEnv map[string]string Runner CommandRunner + CaseEnvironmentFactory CaseEnvironmentFactory + CaseDatabaseIsolation bool } // CoreScenarios returns the v1.7 Step 4 real-world benchmark cases. @@ -418,11 +420,18 @@ func runColdkeep(ctx BenchmarkContext, cfg ScenarioConfig, args ...string) error return fmt.Errorf("coldkeep executable cannot be empty") } + extraEnv := make(map[string]string, len(cfg.ExtraEnv)+len(ctx.ExtraEnv)) + for key, value := range cfg.ExtraEnv { + extraEnv[key] = value + } + for key, value := range ctx.ExtraEnv { + extraEnv[key] = value + } env := withScenarioEnv(os.Environ(), map[string]string{ "COLDKEEP_STORAGE_DIR": filepath.Join(ctx.RepoPath, "storage", "containers"), "COLDKEEP_CODEC": cfg.Codec, "COLDKEEP_COMPRESSION": cfg.Compression, - }, cfg.ExtraEnv) + }, extraEnv) spec := CommandSpec{ Executable: cfg.ColdkeepExecutable, diff --git a/internal/catalog/backend_contract_backends_test.go b/internal/catalog/backend_contract_backends_test.go index 28f73051..ef23d25c 100644 --- a/internal/catalog/backend_contract_backends_test.go +++ b/internal/catalog/backend_contract_backends_test.go @@ -1,141 +1,16 @@ package catalog_test import ( - "database/sql" - "fmt" - "os" "testing" - "time" - _ "github.com/mattn/go-sqlite3" - - "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" ) -// catalogBackend describes a database backend the catalog contract tests run -// against. SQLite always runs; PostgreSQL runs only when COLDKEEP_TEST_DB is set -// (the project-wide convention) and skips cleanly otherwise. -type catalogBackend struct { - Name string - Open func(t *testing.T) *sql.DB -} - -// catalogBackends returns the backends the dual-backend contract suite exercises. -// The SQLite backend is unconditional. The PostgreSQL backend follows the -// existing project convention used by internal/db/migrations_test.go: it reads a -// DSN from the environment and is gated by COLDKEEP_TEST_DB. CI provides a -// postgres service and sets COLDKEEP_TEST_DB=1, so the PostgreSQL path runs in CI -// even though it skips during local development without a configured database. -func catalogBackends() []catalogBackend { - return []catalogBackend{ - {Name: "sqlite", Open: openSQLiteCatalogTestDB}, - {Name: "postgres", Open: openPostgresCatalogTestDBOrSkip}, - } -} - -// openSQLiteCatalogTestDB opens an in-memory SQLite database with the coldkeep -// schema applied. It mirrors the openTestDB helper used by the other catalog -// tests and is always available. -func openSQLiteCatalogTestDB(t *testing.T) *sql.DB { - t.Helper() - dbconn, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("sql.Open sqlite3: %v", err) - } - t.Cleanup(func() { _ = dbconn.Close() }) - if err := db.RunMigrations(dbconn); err != nil { - t.Fatalf("RunMigrations: %v", err) - } - return dbconn -} - -// openPostgresCatalogTestDBOrSkip opens the configured PostgreSQL test database -// and applies the coldkeep schema to it. The fixture seeding is idempotent, so -// this helper provisions a fresh temporary database per test so idempotent -// fixture inserts cannot leak state between PostgreSQL subtests. -func openPostgresCatalogTestDBOrSkip(t *testing.T) *sql.DB { - t.Helper() - if os.Getenv("COLDKEEP_TEST_DB") == "" { - t.Skip("Set COLDKEEP_TEST_DB=1 (with DB_* DSN env) to run PostgreSQL catalog contract tests") - } - - cfg := loadPostgresCatalogTestConfig() - adminDB := openPostgresCatalogAdminConnection(t, cfg) - testDBName := fmt.Sprintf("coldkeep_catalog_contract_%d", time.Now().UnixNano()) - if _, err := adminDB.Exec(fmt.Sprintf("CREATE DATABASE %s", testDBName)); err != nil { - t.Fatalf("create temporary postgres catalog database %s: %v", testDBName, err) - } - t.Cleanup(func() { - _, _ = adminDB.Exec(` - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = $1 AND pid <> pg_backend_pid() - `, testDBName) - _, _ = adminDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", testDBName)) - _ = adminDB.Close() - }) - - dbconn := openPostgresCatalogTestConnection(t, cfg, testDBName, "test database") - - t.Setenv("COLDKEEP_DB_AUTO_BOOTSTRAP", "true") - if err := db.EnsurePostgresSchema(dbconn); err != nil { - _ = dbconn.Close() - t.Fatalf("apply postgres schema to %s: %v", cfg.Database, err) - } - - t.Cleanup(func() { _ = dbconn.Close() }) - return dbconn -} - -func openPostgresCatalogAdminConnection(t *testing.T, cfg postgresCatalogTestConfig) *sql.DB { +// forEachCatalogBackend delegates fixture lifecycle to the reusable harness. +// PostgreSQL remains optional locally. Required CI is configured to execute +// its PostgreSQL subtests in correctness-matrix's plain-codec package-contract +// step. +func forEachCatalogBackend(t *testing.T, fn func(t *testing.T, backend backendtest.Backend)) { t.Helper() - maintenanceDB := getenvOrDefaultCatalogTest("COLDKEEP_TEST_DB_MAINTENANCE", "postgres") - return openPostgresCatalogTestConnection(t, cfg, maintenanceDB, "admin") -} - -type postgresCatalogTestConfig struct { - Host string - Port string - User string - Password string - SSLMode string - Database string -} - -func loadPostgresCatalogTestConfig() postgresCatalogTestConfig { - return postgresCatalogTestConfig{ - Host: getenvOrDefaultCatalogTest("DB_HOST", "127.0.0.1"), - Port: getenvOrDefaultCatalogTest("DB_PORT", "5432"), - User: getenvOrDefaultCatalogTest("DB_USER", "coldkeep"), - Password: getenvOrDefaultCatalogTest("DB_PASSWORD", "coldkeep"), - SSLMode: getenvOrDefaultCatalogTest("DB_SSLMODE", "disable"), - Database: getenvOrDefaultCatalogTest("DB_NAME", "coldkeep"), - } -} - -func openPostgresCatalogTestConnection(t *testing.T, cfg postgresCatalogTestConfig, databaseName, purpose string) *sql.DB { - t.Helper() - dbconn, err := sql.Open("postgres", postgresCatalogTestConnString(cfg, databaseName)) - if err != nil { - t.Fatalf("open postgres %s connection: %v", purpose, err) - } - if err := dbconn.Ping(); err != nil { - _ = dbconn.Close() - t.Fatalf("ping postgres %s connection: %v", purpose, err) - } - return dbconn -} - -func postgresCatalogTestConnString(cfg postgresCatalogTestConfig, databaseName string) string { - return fmt.Sprintf( - "host=%s port=%s user=%s password=%s dbname=%s sslmode=%s connect_timeout=5", - cfg.Host, cfg.Port, cfg.User, cfg.Password, databaseName, cfg.SSLMode, - ) -} - -func getenvOrDefaultCatalogTest(key, fallback string) string { - if v, ok := os.LookupEnv(key); ok && v != "" { - return v - } - return fallback + backendtest.ForEach(t, backendtest.Options{}, fn) } diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index 34cff890..ddc02bda 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -4,577 +4,403 @@ import ( "context" "database/sql" "errors" + "reflect" "testing" "time" "github.com/franchoy/coldkeep/internal/catalog" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" ) -// catalogFixtureBase is the fixed UTC base timestamp used by the fixture so that -// timestamp behavior is deterministic across SQLite and PostgreSQL. var catalogFixtureBase = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) -// seedCatalogFixture inserts an identical logical fixture into either backend -// using backend-neutral SQL. All values are bound through $1-style placeholders -// with appropriate Go types (bool for the boolean column, time.Time for -// timestamps) so neither backend's literal conventions leak in. -// -// Fixture shape: -// -// logical_file: -// 1 current-file.txt COMPLETED size=11 hash=h1 -// 2 snapshot-only-file COMPLETED size=22 hash=h2 -// physical_file: -// /current/a.txt -> lf 1 (mtime set, is_metadata_complete=true) -// /current/b.txt -> lf 1 (mtime NULL, is_metadata_complete=false) -// snapshot: -// snap-full (full, label "alpha", created base) -// snap-child (partial, label "beta", created base+1h, parent snap-full) -// snapshot_path: -// /snapshot/file.txt -// snapshot_file: -// snap-full -> path -> lf 2 -func seedCatalogFixture(t *testing.T, dbconn *sql.DB) { - t.Helper() - exec := newCatalogFixtureExec(t, dbconn) - seedCatalogLogicalFiles(exec) - seedCatalogPhysicalFiles(exec) - seedCatalogSnapshots(exec) - seedCatalogSnapshotFiles(exec) -} +const catalogFixtureLargeID int64 = 4_000_000_000 -type catalogFixtureExec func(string, ...any) - -func newCatalogFixtureExec(t *testing.T, dbconn *sql.DB) catalogFixtureExec { +// seedCatalogFixture is one backend-neutral fixture for every CAT contract. +// It deliberately includes null values, duplicate root inputs, equal snapshot +// timestamps, both reachability sources, and an unreferenced logical file. +func seedCatalogFixture(t *testing.T, dbconn *sql.DB) { t.Helper() - ctx := context.Background() - return func(query string, args ...any) { + exec := func(query string, args ...any) { t.Helper() - if _, err := dbconn.ExecContext(ctx, query, args...); err != nil { + if _, err := dbconn.ExecContext(context.Background(), query, args...); err != nil { t.Fatalf("seed exec failed: %v\nquery: %s", err, query) } } -} -func seedCatalogLogicalFiles(exec catalogFixtureExec) { - exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (id) DO UPDATE SET - original_name = EXCLUDED.original_name, - total_size = EXCLUDED.total_size, - file_hash = EXCLUDED.file_hash, - ref_count = EXCLUDED.ref_count, - status = EXCLUDED.status`, - 1, "current-file.txt", 11, "h1", 1, "COMPLETED") - exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (id) DO UPDATE SET - original_name = EXCLUDED.original_name, - total_size = EXCLUDED.total_size, - file_hash = EXCLUDED.file_hash, - ref_count = EXCLUDED.ref_count, - status = EXCLUDED.status`, - 2, "snapshot-only-file.txt", 22, "h2", 1, "COMPLETED") -} + logicalFiles := []struct { + id int64 + name string + size int64 + hash string + refCount int + status string + }{ + {1, "current-file.txt", 11, "h1", 7, "COMPLETED"}, + {2, "snapshot-only-file.txt", 22, "h2", 1, "COMPLETED"}, + {3, "both-roots-file.txt", 33, "h3", 2, "COMPLETED"}, + {4, "unreferenced-file.txt", 44, "h4", 0, "ABORTED"}, + {5, "incomplete-current-file.txt", 55, "h5", 1, "PROCESSING"}, + {catalogFixtureLargeID, "large-id-file.txt", 4_000_000_001, "h-large", 9, "COMPLETED"}, + } + for _, row := range logicalFiles { + exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) + VALUES ($1, $2, $3, $4, $5, $6)`, + row.id, row.name, row.size, row.hash, row.refCount, row.status) + } -func seedCatalogPhysicalFiles(exec catalogFixtureExec) { - // Physical file with full metadata (non-null mtime, is_metadata_complete=true). exec(`INSERT INTO physical_file (path, logical_file_id, mode, mtime, is_metadata_complete) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (path) DO UPDATE SET - logical_file_id = EXCLUDED.logical_file_id, - mode = EXCLUDED.mode, - mtime = EXCLUDED.mtime, - is_metadata_complete = EXCLUDED.is_metadata_complete`, - "/current/a.txt", 1, 0o644, catalogFixtureBase, true) - // Physical file with NULL mtime and is_metadata_complete=false (nullable case). + VALUES ($1, $2, $3, $4, $5)`, "/current/a.txt", 1, 0o644, catalogFixtureBase, true) exec(`INSERT INTO physical_file (path, logical_file_id, mode, mtime, is_metadata_complete) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (path) DO UPDATE SET - logical_file_id = EXCLUDED.logical_file_id, - mode = EXCLUDED.mode, - mtime = EXCLUDED.mtime, - is_metadata_complete = EXCLUDED.is_metadata_complete`, - "/current/b.txt", 1, nil, nil, false) -} - -func seedCatalogSnapshots(exec catalogFixtureExec) { - exec(`INSERT INTO snapshot (id, created_at, type, label) VALUES ($1, $2, $3, $4) - ON CONFLICT (id) DO UPDATE SET - created_at = EXCLUDED.created_at, - type = EXCLUDED.type, - label = EXCLUDED.label, - parent_id = NULL`, - "snap-full", catalogFixtureBase, "full", "alpha") - exec(`INSERT INTO snapshot (id, created_at, type, label, parent_id) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (id) DO UPDATE SET - created_at = EXCLUDED.created_at, - type = EXCLUDED.type, - label = EXCLUDED.label, - parent_id = EXCLUDED.parent_id`, - "snap-child", catalogFixtureBase.Add(time.Hour), "partial", "beta", "snap-full") -} - -func seedCatalogSnapshotFiles(exec catalogFixtureExec) { - exec(`INSERT INTO snapshot_path (id, path) VALUES ($1, $2) - ON CONFLICT (id) DO UPDATE SET path = EXCLUDED.path`, - 1, "/snapshot/file.txt") - exec(`INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id) VALUES ($1, $2, $3) - ON CONFLICT (snapshot_id, path_id) DO UPDATE SET logical_file_id = EXCLUDED.logical_file_id`, - "snap-full", 1, 2) -} - -type logicalFileFinder interface { - FindLogicalFile(context.Context, int64) (*catalog.LogicalFileRef, error) -} - -// TestCatalogContractFindLogicalFileAcrossBackends verifies FindLogicalFile -// returns identical results on every backend. + VALUES ($1, $2, $3, $4, $5)`, "/current/b.txt", 1, nil, nil, false) + exec(`INSERT INTO physical_file (path, logical_file_id, mode, mtime, is_metadata_complete) + VALUES ($1, $2, $3, $4, $5)`, "/current/both.txt", 3, 0o600, catalogFixtureBase.Add(time.Minute), true) + exec(`INSERT INTO physical_file (path, logical_file_id, mode, mtime, is_metadata_complete) + VALUES ($1, $2, $3, $4, $5)`, "/current/incomplete.txt", 5, nil, nil, false) + + type snapshotRow struct { + id, typ string + created time.Time + label, parent any + } + for _, row := range []snapshotRow{ + {"snap-full", "full", catalogFixtureBase, "alpha", nil}, + {"snap-tie-a", "full", catalogFixtureBase, "tie-a", nil}, + {"snap-tie-b", "full", catalogFixtureBase, "tie-b", nil}, + {"snap-child", "partial", catalogFixtureBase.Add(time.Hour), "beta", "snap-full"}, + {"snap-null-label", "full", catalogFixtureBase.Add(2 * time.Hour), nil, nil}, + } { + exec(`INSERT INTO snapshot (id, created_at, type, label, parent_id) + VALUES ($1, $2, $3, $4, $5)`, row.id, row.created, row.typ, row.label, row.parent) + } + for _, row := range []struct { + id int64 + path string + }{ + {1, "/snapshot/one.txt"}, + {2, "/snapshot/two.txt"}, + {3, "/snapshot/both.txt"}, + } { + exec(`INSERT INTO snapshot_path (id, path) VALUES ($1, $2)`, row.id, row.path) + } + for _, row := range []struct { + snapshotID string + pathID, logicalFileID int64 + }{ + {"snap-full", 1, 2}, + {"snap-tie-a", 2, 2}, + {"snap-child", 3, 3}, + } { + exec(`INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id) VALUES ($1, $2, $3)`, + row.snapshotID, row.pathID, row.logicalFileID) + } +} + +// CAT-001 proves logical-file lookup, missing results, large int64 values, +// deterministic reads, and non-mutation on both backends. func TestCatalogContractFindLogicalFileAcrossBackends(t *testing.T) { - for _, backend := range catalogBackends() { - t.Run(backend.Name, func(t *testing.T) { - dbconn := backend.Open(t) - seedCatalogFixture(t, dbconn) - assertCatalogFindLogicalFile(t, catalog.NewServiceFromSQL(dbconn)) - }) - } -} - -func assertCatalogFindLogicalFile(t *testing.T, svc logicalFileFinder) { - t.Helper() - ctx := context.Background() - assertMissingLogicalFile(t, svc, ctx, 9999) - assertLogicalFileRef(t, svc, ctx, 1) -} - -func assertMissingLogicalFile(t *testing.T, svc logicalFileFinder, ctx context.Context, id int64) { - t.Helper() - missing, err := svc.FindLogicalFile(ctx, id) - if err != nil { - t.Fatalf("FindLogicalFile(missing): %v", err) - } - if missing != nil { - t.Fatalf("FindLogicalFile(missing): want nil, got %+v", missing) - } -} - -func assertLogicalFileRef(t *testing.T, svc logicalFileFinder, ctx context.Context, id int64) { - t.Helper() - got := requireLogicalFileRef(t, svc, ctx, id) - assertLogicalFileFields(t, got, expectedLogicalFileRef()) + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + assertNilLogicalFile(t, svc, 9999) + assertLogicalFile(t, svc, 1, catalog.LogicalFileRef{ID: 1, OriginalName: "current-file.txt", TotalSize: 11, FileHash: "h1", RefCount: 7, Status: "COMPLETED"}) + assertLogicalFile(t, svc, catalogFixtureLargeID, catalog.LogicalFileRef{ID: catalogFixtureLargeID, OriginalName: "large-id-file.txt", TotalSize: 4_000_000_001, FileHash: "h-large", RefCount: 9, Status: "COMPLETED"}) + assertCancelledCatalogErrors(t, svc, backend.DB) + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("CAT-001 reads mutated catalog state: before=%+v after=%+v", before, after) + } + }) } -func requireLogicalFileRef(t *testing.T, svc logicalFileFinder, ctx context.Context, id int64) *catalog.LogicalFileRef { +func assertNilLogicalFile(t *testing.T, svc interface { + FindLogicalFile(context.Context, int64) (*catalog.LogicalFileRef, error) +}, id int64) { t.Helper() - got, err := svc.FindLogicalFile(ctx, id) - if err != nil { - t.Fatalf("FindLogicalFile(%d): %v", id, err) - } - if got == nil { - t.Fatalf("FindLogicalFile(%d): want ref, got nil", id) + got, err := svc.FindLogicalFile(context.Background(), id) + if err != nil || got != nil { + t.Fatalf("FindLogicalFile(%d): got (%+v, %v), want (nil, nil)", id, got, err) } - return got } -func expectedLogicalFileRef() catalog.LogicalFileRef { - return catalog.LogicalFileRef{ - ID: 1, - OriginalName: "current-file.txt", - TotalSize: 11, - FileHash: "h1", - RefCount: 1, - Status: "COMPLETED", - } -} - -func assertLogicalFileFields(t *testing.T, got *catalog.LogicalFileRef, want catalog.LogicalFileRef) { +func assertLogicalFile(t *testing.T, svc interface { + FindLogicalFile(context.Context, int64) (*catalog.LogicalFileRef, error) +}, id int64, want catalog.LogicalFileRef) { t.Helper() - if got.ID != want.ID { - t.Errorf("ID: got %d, want %d", got.ID, want.ID) - } - if got.OriginalName != want.OriginalName { - t.Errorf("OriginalName: got %q, want %q", got.OriginalName, want.OriginalName) - } - if got.TotalSize != want.TotalSize { - t.Errorf("TotalSize: got %d, want %d", got.TotalSize, want.TotalSize) - } - if got.FileHash != want.FileHash { - t.Errorf("FileHash: got %q, want %q", got.FileHash, want.FileHash) + first, err := svc.FindLogicalFile(context.Background(), id) + if err != nil || first == nil { + t.Fatalf("FindLogicalFile(%d): got (%+v, %v)", id, first, err) } - if got.RefCount != want.RefCount { - t.Errorf("RefCount: got %d, want %d", got.RefCount, want.RefCount) + second, err := svc.FindLogicalFile(context.Background(), id) + if err != nil || second == nil { + t.Fatalf("FindLogicalFile(%d) repeated: got (%+v, %v)", id, second, err) } - if got.Status != want.Status { - t.Errorf("Status: got %q, want %q", got.Status, want.Status) + if !reflect.DeepEqual(*first, want) || !reflect.DeepEqual(*second, want) { + t.Fatalf("FindLogicalFile(%d): got %+v then %+v, want %+v", id, first, second, want) } } -// TestCatalogContractFindPhysicalFilesAcrossBackends verifies ordering, nullable -// metadata, and boolean handling (the most common SQLite/PostgreSQL trap) are -// consistent across backends. +// CAT-002 proves deterministic physical-file ordering and nullable/boolean +// semantics. The public contract normalizes a NULL mode to zero. func TestCatalogContractFindPhysicalFilesAcrossBackends(t *testing.T) { - for _, backend := range catalogBackends() { - t.Run(backend.Name, func(t *testing.T) { - dbconn := backend.Open(t) - seedCatalogFixture(t, dbconn) - assertCatalogFindPhysicalFiles(t, catalog.NewServiceFromSQL(dbconn)) - }) - } + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + empty, err := svc.FindPhysicalFilesForLogicalFile(context.Background(), 9999) + if err != nil || len(empty) != 0 { + t.Fatalf("FindPhysicalFilesForLogicalFile(missing): got (%+v, %v), want empty nil-error result", empty, err) + } + refs := requirePhysicalFiles(t, svc, 1) + if got := []string{refs[0].Path, refs[1].Path}; !reflect.DeepEqual(got, []string{"/current/a.txt", "/current/b.txt"}) { + t.Fatalf("physical-file ordering: got %v", got) + } + if refs[0].LogicalFileID != 1 || refs[0].Mode != 0o644 || refs[0].MTime == nil || !refs[0].MTime.Equal(catalogFixtureBase) || !refs[0].IsMetadataComplete { + t.Fatalf("complete physical file: got %+v", refs[0]) + } + if refs[1].LogicalFileID != 1 || refs[1].Mode != 0 || refs[1].MTime != nil || refs[1].IsMetadataComplete { + t.Fatalf("incomplete physical file: got %+v", refs[1]) + } + if repeated := requirePhysicalFiles(t, svc, 1); !reflect.DeepEqual(refs, repeated) { + t.Fatalf("physical-file reads are not deterministic: first=%+v repeated=%+v", refs, repeated) + } + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("CAT-002 reads mutated catalog state: before=%+v after=%+v", before, after) + } + }) } -type physicalFileFinder interface { +func requirePhysicalFiles(t *testing.T, svc interface { FindPhysicalFilesForLogicalFile(context.Context, int64) ([]catalog.PhysicalFileRef, error) -} - -func assertCatalogFindPhysicalFiles(t *testing.T, svc physicalFileFinder) { - t.Helper() - ctx := context.Background() - assertMissingPhysicalFiles(t, svc, ctx, 9999) - refs := requirePhysicalFiles(t, svc, ctx, 1, 2) - assertPhysicalFileOrdering(t, refs) - assertCompletePhysicalFile(t, refs[0]) - assertIncompletePhysicalFile(t, refs[1]) -} - -func assertMissingPhysicalFiles(t *testing.T, svc physicalFileFinder, ctx context.Context, id int64) { - t.Helper() - refs := requirePhysicalFiles(t, svc, ctx, id, 0) - if len(refs) != 0 { - t.Fatalf("FindPhysicalFilesForLogicalFile(missing): want empty, got %d", len(refs)) - } -} - -func requirePhysicalFiles( - t *testing.T, - svc physicalFileFinder, - ctx context.Context, - id int64, - wantRows int, -) []catalog.PhysicalFileRef { +}, id int64) []catalog.PhysicalFileRef { t.Helper() - refs, err := svc.FindPhysicalFilesForLogicalFile(ctx, id) - if err != nil { - t.Fatalf("FindPhysicalFilesForLogicalFile(%d): %v", id, err) - } - if len(refs) != wantRows { - t.Fatalf("FindPhysicalFilesForLogicalFile(%d): want %d rows, got %d", id, wantRows, len(refs)) + refs, err := svc.FindPhysicalFilesForLogicalFile(context.Background(), id) + if err != nil || len(refs) != 2 { + t.Fatalf("FindPhysicalFilesForLogicalFile(%d): got (%+v, %v), want two rows", id, refs, err) } return refs } -func assertPhysicalFileOrdering(t *testing.T, refs []catalog.PhysicalFileRef) { - t.Helper() - if refs[0].Path != "/current/a.txt" || refs[1].Path != "/current/b.txt" { - t.Fatalf("ordering: got %q then %q", refs[0].Path, refs[1].Path) - } -} - -func assertCompletePhysicalFile(t *testing.T, ref catalog.PhysicalFileRef) { - t.Helper() - if ref.MTime == nil { - t.Errorf("row a: expected non-nil MTime") - } else if !ref.MTime.Equal(catalogFixtureBase) { - t.Errorf("row a: MTime = %v, want %v", ref.MTime, catalogFixtureBase) - } - if !ref.IsMetadataComplete { - t.Errorf("row a: IsMetadataComplete = false, want true") - } +// CAT-003 proves snapshot lookup of root/child/null-label records, timestamp +// normalization, stable missing results, and repeatability. +func TestCatalogContractFindSnapshotAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + missing, err := svc.FindSnapshot(context.Background(), "does-not-exist") + if err != nil || missing != nil { + t.Fatalf("FindSnapshot(missing): got (%+v, %v), want (nil, nil)", missing, err) + } + assertSnapshot(t, svc, "snap-full", "full", "alpha", "", catalogFixtureBase) + assertSnapshot(t, svc, "snap-child", "partial", "beta", "snap-full", catalogFixtureBase.Add(time.Hour)) + assertSnapshot(t, svc, "snap-null-label", "full", "", "", catalogFixtureBase.Add(2*time.Hour)) + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("CAT-003 reads mutated catalog state: before=%+v after=%+v", before, after) + } + }) } -func assertIncompletePhysicalFile(t *testing.T, ref catalog.PhysicalFileRef) { +func assertSnapshot(t *testing.T, svc interface { + FindSnapshot(context.Context, string) (*catalog.SnapshotRef, error) +}, id, typ, label, parent string, created time.Time) { t.Helper() - if ref.MTime != nil { - t.Errorf("row b: expected nil MTime for NULL, got %v", ref.MTime) + first, err := svc.FindSnapshot(context.Background(), id) + if err != nil || first == nil { + t.Fatalf("FindSnapshot(%q): got (%+v, %v)", id, first, err) } - if ref.IsMetadataComplete { - t.Errorf("row b: IsMetadataComplete = true, want false") + second, err := svc.FindSnapshot(context.Background(), id) + if err != nil || !reflect.DeepEqual(first, second) { + t.Fatalf("FindSnapshot(%q) repeat: got (%+v, %v), first=%+v", id, second, err, first) } -} - -// TestCatalogContractFindSnapshotAcrossBackends verifies snapshot identity, -// nullable parent/label, and timestamp parsing are consistent across backends. -func TestCatalogContractFindSnapshotAcrossBackends(t *testing.T) { - for _, backend := range catalogBackends() { - t.Run(backend.Name, func(t *testing.T) { - dbconn := backend.Open(t) - seedCatalogFixture(t, dbconn) - assertCatalogFindSnapshot(t, catalog.NewServiceFromSQL(dbconn)) - }) + if first.ID != id || first.Type != typ || first.Label != label || first.ParentID != parent || !first.CreatedAt.Equal(created) { + t.Fatalf("FindSnapshot(%q): got %+v", id, first) } } -type snapshotFinder interface { - FindSnapshot(context.Context, string) (*catalog.SnapshotRef, error) -} - -func assertCatalogFindSnapshot(t *testing.T, svc snapshotFinder) { - t.Helper() - ctx := context.Background() - assertMissingCatalogSnapshot(t, svc, ctx, "does-not-exist") - assertRootCatalogSnapshot(t, requireCatalogSnapshot(t, svc, ctx, "snap-full")) - assertChildCatalogSnapshot(t, requireCatalogSnapshot(t, svc, ctx, "snap-child")) +// CAT-004 proves list ordering, equal-time tie breaking, filters, inclusive +// time bounds, ordinary literal substring behavior, and limit semantics. +func TestCatalogContractListSnapshotsAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{}, "snap-null-label", "snap-child", "snap-tie-b", "snap-tie-a", "snap-full") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{Type: "full", LabelSubstring: "tie"}, "snap-tie-b", "snap-tie-a") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{LabelSubstring: "bet"}, "snap-child") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{Since: timePointer(catalogFixtureBase.Add(time.Hour))}, "snap-null-label", "snap-child") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{Until: timePointer(catalogFixtureBase)}, "snap-tie-b", "snap-tie-a", "snap-full") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{Limit: 2}, "snap-null-label", "snap-child") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{Limit: 0}, "snap-null-label", "snap-child", "snap-tie-b", "snap-tie-a", "snap-full") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{Limit: -1}, "snap-null-label", "snap-child", "snap-tie-b", "snap-tie-a", "snap-full") + assertSnapshotIDs(t, svc, catalog.SnapshotFilter{LabelSubstring: "absent"}) + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("CAT-004 reads mutated catalog state: before=%+v after=%+v", before, after) + } + }) } -func assertMissingCatalogSnapshot(t *testing.T, svc snapshotFinder, ctx context.Context, id string) { - t.Helper() - missing, err := svc.FindSnapshot(ctx, id) - if err != nil { - t.Fatalf("FindSnapshot(missing): %v", err) - } - if missing != nil { - t.Fatalf("FindSnapshot(missing): want nil, got %+v", missing) - } -} +func timePointer(value time.Time) *time.Time { return &value } -func requireCatalogSnapshot(t *testing.T, svc snapshotFinder, ctx context.Context, id string) *catalog.SnapshotRef { +func assertSnapshotIDs(t *testing.T, svc interface { + ListSnapshots(context.Context, catalog.SnapshotFilter) ([]catalog.SnapshotRef, error) +}, filter catalog.SnapshotFilter, want ...string) { t.Helper() - ref, err := svc.FindSnapshot(ctx, id) + refs, err := svc.ListSnapshots(context.Background(), filter) if err != nil { - t.Fatalf("FindSnapshot(%s): %v", id, err) - } - if ref == nil { - t.Fatalf("FindSnapshot(%s): want ref, got nil", id) + t.Fatalf("ListSnapshots(%+v): %v", filter, err) } - return ref -} - -func assertRootCatalogSnapshot(t *testing.T, ref *catalog.SnapshotRef) { - t.Helper() - if ref.ID != "snap-full" || ref.Type != "full" || ref.Label != "alpha" { - t.Fatalf("FindSnapshot(snap-full): unexpected ref %+v", ref) + got := make([]string, len(refs)) + for i, ref := range refs { + got[i] = ref.ID } - if ref.ParentID != "" { - t.Errorf("FindSnapshot(snap-full): ParentID = %q, want empty", ref.ParentID) + if len(want) == 0 && len(got) == 0 { + return } - if !ref.CreatedAt.Equal(catalogFixtureBase) { - t.Errorf("FindSnapshot(snap-full): CreatedAt = %v, want %v", ref.CreatedAt, catalogFixtureBase) - } -} - -func assertChildCatalogSnapshot(t *testing.T, ref *catalog.SnapshotRef) { - t.Helper() - if ref.Type != "partial" || ref.Label != "beta" || ref.ParentID != "snap-full" { - t.Fatalf("FindSnapshot(snap-child): unexpected ref %+v", ref) + if !reflect.DeepEqual(got, want) { + t.Fatalf("ListSnapshots(%+v): got %v, want %v", filter, got, want) } -} - -type snapshotLister interface { - ListSnapshots(context.Context, catalog.SnapshotFilter) ([]catalog.SnapshotRef, error) -} - -// TestCatalogContractListSnapshotsAcrossBackends verifies ordering (newest -// first), type filtering, label substring matching (LIKE), Since/Until bounds, -// and Limit are consistent across backends. -func TestCatalogContractListSnapshotsAcrossBackends(t *testing.T) { - for _, backend := range catalogBackends() { - t.Run(backend.Name, func(t *testing.T) { - dbconn := backend.Open(t) - seedCatalogFixture(t, dbconn) - assertCatalogListSnapshots(t, catalog.NewServiceFromSQL(dbconn)) - }) + for _, ref := range refs { + if ref.ID == "snap-null-label" && ref.Label != "" { + t.Fatalf("ListSnapshots null label: got %q, want public empty representation", ref.Label) + } } } -func assertCatalogListSnapshots(t *testing.T, svc snapshotLister) { - t.Helper() - ctx := context.Background() - assertAllSnapshots(t, svc, ctx) - assertFilteredSnapshot(t, svc, ctx, catalog.SnapshotFilter{Type: "full"}, "type=full", "snap-full") - assertFilteredSnapshot(t, svc, ctx, catalog.SnapshotFilter{LabelSubstring: "alph"}, "label~alph", "snap-full") - assertSnapshotTimeBounds(t, svc, ctx) - assertFilteredSnapshot(t, svc, ctx, catalog.SnapshotFilter{Limit: 1}, "limit=1", "snap-child") +// CAT-005 proves GC-safety root sets are unique, separated by source, include +// a legitimately both-reachable file in both sets, and are independent maps. +func TestCatalogContractLoadReachabilityRootsAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + roots, err := svc.LoadReachabilityRoots(context.Background()) + if err != nil || roots == nil { + t.Fatalf("LoadReachabilityRoots: got (%+v, %v)", roots, err) + } + assertIDSet(t, roots.Current, 1, 3, 5) + assertIDSet(t, roots.Snapshot, 2, 3) + if _, ok := roots.Current[4]; ok { + t.Fatal("unreferenced logical file appears in current roots") + } + if _, ok := roots.Snapshot[1]; ok { + t.Fatal("current-only logical file appears in snapshot roots") + } + roots.Current[99] = struct{}{} + if _, ok := roots.Snapshot[99]; ok { + t.Fatal("current and snapshot root maps alias each other") + } + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("CAT-005 reads mutated catalog state: before=%+v after=%+v", before, after) + } + }) } -func assertAllSnapshots(t *testing.T, svc snapshotLister, ctx context.Context) { +func assertIDSet(t *testing.T, got map[int64]struct{}, want ...int64) { t.Helper() - all := requireSnapshots(t, svc, ctx, catalog.SnapshotFilter{}, "all") - if len(all) != 2 { - t.Fatalf("ListSnapshots(all): want 2, got %d", len(all)) + if len(got) != len(want) { + t.Fatalf("root set length: got %d, want %d (%v)", len(got), len(want), want) } - if all[0].ID != "snap-child" || all[1].ID != "snap-full" { - t.Fatalf("ListSnapshots(all): ordering got %q then %q", all[0].ID, all[1].ID) + for _, id := range want { + if _, ok := got[id]; !ok { + t.Fatalf("root set missing %d: got %v", id, got) + } } } -func assertSnapshotTimeBounds(t *testing.T, svc snapshotLister, ctx context.Context) { - t.Helper() - since := catalogFixtureBase.Add(30 * time.Minute) - assertFilteredSnapshot(t, svc, ctx, catalog.SnapshotFilter{Since: &since}, "since", "snap-child") - - until := catalogFixtureBase.Add(30 * time.Minute) - assertFilteredSnapshot(t, svc, ctx, catalog.SnapshotFilter{Until: &until}, "until", "snap-full") +// CAT-006 preserves the deliberately deferred API boundary and proves those +// methods cannot return partial results or mutate the catalog. +func TestCatalogContractDeferredMethodsAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + graph, err := svc.LoadSnapshotGraph(context.Background()) + assertDeferred(t, "LoadSnapshotGraph", err, graph) + placements, err := svc.LoadChunkPlacements(context.Background(), 1) + assertDeferred(t, "LoadChunkPlacements", err, placements) + restorePlan, err := svc.LoadRestorePlanMetadata(context.Background(), catalog.RestorePlanInput{FileID: 1}) + assertDeferred(t, "LoadRestorePlanMetadata", err, restorePlan) + gcPlan, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{}) + assertDeferred(t, "LoadGCPlanMetadata", err, gcPlan) + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("deferred catalog methods mutated catalog state: before=%+v after=%+v", before, after) + } + }) } -func assertFilteredSnapshot( - t *testing.T, - svc snapshotLister, - ctx context.Context, - filter catalog.SnapshotFilter, - label string, - wantID string, -) { +func assertDeferred(t *testing.T, name string, err error, result any) { t.Helper() - refs := requireSnapshots(t, svc, ctx, filter, label) - if len(refs) != 1 || refs[0].ID != wantID { - t.Fatalf("ListSnapshots(%s): got %+v", label, refs) + if !errors.Is(err, catalog.ErrNotImplemented) || !catalog.IsDeferred(err) { + t.Errorf("%s: want catalog.ErrNotImplemented, got %v", name, err) } -} - -func requireSnapshots( - t *testing.T, - svc snapshotLister, - ctx context.Context, - filter catalog.SnapshotFilter, - label string, -) []catalog.SnapshotRef { - t.Helper() - refs, err := svc.ListSnapshots(ctx, filter) - if err != nil { - t.Fatalf("ListSnapshots(%s): %v", label, err) + if !isNil(result) { + t.Errorf("%s: want nil result, got %#v", name, result) } - return refs } -// TestCatalogContractLoadReachabilityRootsAcrossBackends verifies the current -// and snapshot reachability sets are populated from the correct sources and -// remain separate. This boundary is safety-critical for Phase 6 GC planning. -func TestCatalogContractLoadReachabilityRootsAcrossBackends(t *testing.T) { - for _, backend := range catalogBackends() { - t.Run(backend.Name, func(t *testing.T) { - dbconn := backend.Open(t) - seedCatalogFixture(t, dbconn) - assertCatalogReachabilityRoots(t, catalog.NewServiceFromSQL(dbconn)) - }) +func isNil(value any) bool { + if value == nil { + return true + } + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Interface: + return v.IsNil() + default: + return false } } -type reachabilityRootLoader interface { - LoadReachabilityRoots(context.Context) (*catalog.ReachabilityRoots, error) -} - -func assertCatalogReachabilityRoots(t *testing.T, svc reachabilityRootLoader) { - t.Helper() - roots := requireCatalogReachabilityRoots(t, svc) - assertCatalogCurrentReachabilityRoots(t, roots.Current) - assertCatalogSnapshotReachabilityRoots(t, roots.Snapshot) -} - -func requireCatalogReachabilityRoots(t *testing.T, svc reachabilityRootLoader) *catalog.ReachabilityRoots { +// CAT-007 adds bounded portable cancelled-context assertions. Errors must not +// be converted to not-found results, and cancelled reads must not mutate state. +func assertCancelledCatalogErrors(t *testing.T, svc catalog.Catalog, dbconn *sql.DB) { t.Helper() - roots, err := svc.LoadReachabilityRoots(context.Background()) - if err != nil { - t.Fatalf("LoadReachabilityRoots: %v", err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + before := catalogStateCounts(t, dbconn) + if ref, err := svc.FindLogicalFile(ctx, 1); err == nil || ref != nil { + t.Errorf("cancelled FindLogicalFile: got (%+v, %v), want (nil, error)", ref, err) } - if roots == nil { - t.Fatal("LoadReachabilityRoots: want non-nil") + if refs, err := svc.FindPhysicalFilesForLogicalFile(ctx, 1); err == nil || refs != nil { + t.Errorf("cancelled FindPhysicalFilesForLogicalFile: got (%+v, %v), want (nil, error)", refs, err) } - return roots -} - -func assertCatalogCurrentReachabilityRoots(t *testing.T, current map[int64]struct{}) { - t.Helper() - assertCatalogReachabilityContains(t, current, 1, "Current should contain logical file 1") - assertCatalogReachabilityMissing(t, current, 2, "Current should NOT contain logical file 2 (snapshot-only)") - assertCatalogReachabilitySize(t, current, 1, "Current") -} - -func assertCatalogSnapshotReachabilityRoots(t *testing.T, snapshot map[int64]struct{}) { - t.Helper() - assertCatalogReachabilityContains(t, snapshot, 2, "Snapshot should contain logical file 2") - assertCatalogReachabilityMissing(t, snapshot, 1, "Snapshot should NOT contain logical file 1 (current-only)") - assertCatalogReachabilitySize(t, snapshot, 1, "Snapshot") -} - -func assertCatalogReachabilityContains(t *testing.T, set map[int64]struct{}, id int64, message string) { - t.Helper() - if _, ok := set[id]; !ok { - t.Error(message) + if ref, err := svc.FindSnapshot(ctx, "snap-full"); err == nil || ref != nil { + t.Errorf("cancelled FindSnapshot: got (%+v, %v), want (nil, error)", ref, err) } -} - -func assertCatalogReachabilityMissing(t *testing.T, set map[int64]struct{}, id int64, message string) { - t.Helper() - if _, ok := set[id]; ok { - t.Error(message) + if refs, err := svc.ListSnapshots(ctx, catalog.SnapshotFilter{}); err == nil || refs != nil { + t.Errorf("cancelled ListSnapshots: got (%+v, %v), want (nil, error)", refs, err) } -} - -func assertCatalogReachabilitySize(t *testing.T, set map[int64]struct{}, want int, label string) { - t.Helper() - if len(set) != want { - t.Errorf("%s should hold exactly %d unique id, got %d", label, want, len(set)) + if roots, err := svc.LoadReachabilityRoots(ctx); err == nil || roots != nil { + t.Errorf("cancelled LoadReachabilityRoots: got (%+v, %v), want (nil, error)", roots, err) + } + if after := catalogStateCounts(t, dbconn); after != before { + t.Errorf("cancelled catalog reads mutated catalog state: before=%+v after=%+v", before, after) } } -// TestCatalogContractDeferredMethodsAcrossBackends verifies every deferred -// catalog method returns ErrNotImplemented consistently on both backends, making -// the incomplete boundary explicit rather than silently succeeding. -func TestCatalogContractDeferredMethodsAcrossBackends(t *testing.T) { - for _, backend := range catalogBackends() { - t.Run(backend.Name, func(t *testing.T) { - dbconn := backend.Open(t) - svc := catalog.NewServiceFromSQL(dbconn) - ctx := context.Background() - before := countCatalogLogicalFilesBackend(t, dbconn) - - graph, err := svc.LoadSnapshotGraph(ctx) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadSnapshotGraph: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadSnapshotGraph: want catalog.IsDeferred=true, got %v", err) - } - if graph != nil { - t.Errorf("LoadSnapshotGraph: want nil graph on deferred path, got %+v", graph) - } - - placements, err := svc.LoadChunkPlacements(ctx, 1) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadChunkPlacements: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadChunkPlacements: want catalog.IsDeferred=true, got %v", err) - } - if placements != nil { - t.Errorf("LoadChunkPlacements: want nil placements on deferred path, got %+v", placements) - } - - restorePlan, err := svc.LoadRestorePlanMetadata(ctx, catalog.RestorePlanInput{FileID: 1}) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadRestorePlanMetadata: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadRestorePlanMetadata: want catalog.IsDeferred=true, got %v", err) - } - if restorePlan != nil { - t.Errorf("LoadRestorePlanMetadata: want nil metadata on deferred path, got %+v", restorePlan) - } - - gcPlan, err := svc.LoadGCPlanMetadata(ctx, catalog.GCPlanInput{}) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadGCPlanMetadata: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadGCPlanMetadata: want catalog.IsDeferred=true, got %v", err) - } - if gcPlan != nil { - t.Errorf("LoadGCPlanMetadata: want nil metadata on deferred path, got %+v", gcPlan) - } - - after := countCatalogLogicalFilesBackend(t, dbconn) - if after != before { - t.Fatalf("deferred catalog methods should not mutate logical_file rows: before=%d after=%d", before, after) - } - }) - } +type catalogStateCount struct { + logicalFiles int + physicalFiles int + snapshots int + snapshotFiles int } -func countCatalogLogicalFilesBackend(t *testing.T, dbconn *sql.DB) int { +func catalogStateCounts(t *testing.T, dbconn *sql.DB) catalogStateCount { t.Helper() - - var count int - if err := dbconn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM logical_file`).Scan(&count); err != nil { - t.Fatalf("count logical_file rows: %v", err) + var result catalogStateCount + for _, count := range []struct { + table string + dest *int + }{ + {"logical_file", &result.logicalFiles}, + {"physical_file", &result.physicalFiles}, + {"snapshot", &result.snapshots}, + {"snapshot_file", &result.snapshotFiles}, + } { + if err := dbconn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM `+count.table).Scan(count.dest); err != nil { + t.Fatalf("count %s rows: %v", count.table, err) + } } - return count + return result } diff --git a/internal/catalog/snapshots.go b/internal/catalog/snapshots.go index 874ba386..214e4254 100644 --- a/internal/catalog/snapshots.go +++ b/internal/catalog/snapshots.go @@ -80,7 +80,7 @@ WHERE ($1 = '' OR type = $1) AND ($2 = '' OR label LIKE $2) AND ($3 = 0 OR created_at >= $4) AND ($5 = 0 OR created_at <= $6) -ORDER BY created_at DESC`, values...) +ORDER BY created_at DESC, id DESC`, values...) if err != nil { return nil, fmt.Errorf("catalog: list snapshots: %w", err) } @@ -97,7 +97,7 @@ WHERE ($1 = '' OR type = $1) AND ($2 = '' OR label LIKE $2) AND ($3 = 0 OR created_at >= $4) AND ($5 = 0 OR created_at <= $6) -ORDER BY created_at DESC +ORDER BY created_at DESC, id DESC LIMIT $7`, values...) if err != nil { return nil, fmt.Errorf("catalog: list snapshots: %w", err) diff --git a/internal/cli/render/envelope.go b/internal/cli/render/envelope.go index 06817771..0fbc44d6 100644 --- a/internal/cli/render/envelope.go +++ b/internal/cli/render/envelope.go @@ -1,7 +1,10 @@ package render import ( + "bytes" "encoding/json" + "errors" + "io" "sort" "time" @@ -54,7 +57,15 @@ func toObjectMap(v any) (map[string]any, error) { } var out map[string]any - if err := json.Unmarshal(encoded, &out); err != nil { + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + if err := decoder.Decode(&out); err != nil { + return nil, err + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return nil, errors.New("unexpected trailing JSON value") + } return nil, err } if out == nil { diff --git a/internal/cli/render/envelope_test.go b/internal/cli/render/envelope_test.go new file mode 100644 index 00000000..210713d6 --- /dev/null +++ b/internal/cli/render/envelope_test.go @@ -0,0 +1,121 @@ +package render + +import ( + "encoding/json" + "math" + "strings" + "testing" +) + +func TestToObjectMapPreservesExactJSONNumbers(t *testing.T) { + t.Parallel() + + input := map[string]any{ + "numbers": map[string]int64{ + "zero": 0, + "one": 1, + "negative_one": -1, + "max_int32": 2147483647, + "above_uint32": 4294967296, + "max_safe_integer": 9007199254740991, + "two_to_53": 9007199254740992, + "beyond_safe_integer": 9007199254740993, + "negative_beyond_safe": -9007199254740993, + "max_int64": math.MaxInt64, + }, + "nested": map[string]any{ + "id": int64(9007199254740993), + }, + "items": []any{ + map[string]any{"offset": int64(math.MaxInt64)}, + }, + "float_half": 0.5, + "float_quarter": 1.25, + "ordinary": int64(42), + "null": nil, + "boolean": true, + "string": "unchanged", + } + + converted, err := toObjectMap(input) + if err != nil { + t.Fatalf("toObjectMap: %v", err) + } + + numbers, ok := converted["numbers"].(map[string]any) + if !ok { + t.Fatalf("numbers type=%T want map[string]any", converted["numbers"]) + } + wantNumbers := map[string]string{ + "zero": "0", + "one": "1", + "negative_one": "-1", + "max_int32": "2147483647", + "above_uint32": "4294967296", + "max_safe_integer": "9007199254740991", + "two_to_53": "9007199254740992", + "beyond_safe_integer": "9007199254740993", + "negative_beyond_safe": "-9007199254740993", + "max_int64": "9223372036854775807", + } + for field, want := range wantNumbers { + assertExactJSONNumber(t, numbers[field], want) + } + + nested, ok := converted["nested"].(map[string]any) + if !ok { + t.Fatalf("nested type=%T want map[string]any", converted["nested"]) + } + assertExactJSONNumber(t, nested["id"], "9007199254740993") + + items, ok := converted["items"].([]any) + if !ok || len(items) != 1 { + t.Fatalf("items=%v (%T) want one-element array", converted["items"], converted["items"]) + } + item, ok := items[0].(map[string]any) + if !ok { + t.Fatalf("items[0] type=%T want map[string]any", items[0]) + } + assertExactJSONNumber(t, item["offset"], "9223372036854775807") + + assertExactJSONNumber(t, converted["float_half"], "0.5") + assertExactJSONNumber(t, converted["float_quarter"], "1.25") + assertExactJSONNumber(t, converted["ordinary"], "42") + if converted["null"] != nil { + t.Fatalf("null=%v want nil", converted["null"]) + } + if got, ok := converted["boolean"].(bool); !ok || !got { + t.Fatalf("boolean=%v (%T) want true", converted["boolean"], converted["boolean"]) + } + if got, ok := converted["string"].(string); !ok || got != "unchanged" { + t.Fatalf("string=%v (%T) want unchanged", converted["string"], converted["string"]) + } + + remarshaled, err := json.Marshal(converted) + if err != nil { + t.Fatalf("remarshal converted object: %v", err) + } + for _, token := range []string{ + `"beyond_safe_integer":9007199254740993`, + `"negative_beyond_safe":-9007199254740993`, + `"max_int64":9223372036854775807`, + `"id":9007199254740993`, + `"offset":9223372036854775807`, + } { + if !strings.Contains(string(remarshaled), token) { + t.Fatalf("remarshaled JSON missing exact token %q: %s", token, remarshaled) + } + } +} + +func assertExactJSONNumber(t *testing.T, value any, want string) { + t.Helper() + + number, ok := value.(json.Number) + if !ok { + t.Fatalf("value=%v type=%T want json.Number", value, value) + } + if got := number.String(); got != want { + t.Fatalf("number=%q want %q", got, want) + } +} diff --git a/internal/container/container.go b/internal/container/container.go index 7bb56839..525d9305 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -106,10 +106,23 @@ func openExistingContainer(readonly bool, path string, maxSize int64, fsys fsx.F return nil, err } - if _, err := readAndValidateContainerHeader(f); err != nil { + header, err := readAndValidateContainerHeader(f) + if err != nil { _ = f.Close() return nil, fmt.Errorf("validate container header %s: %w", path, err) } + if maxSize <= ContainerHdrLen { + _ = f.Close() + return nil, fmt.Errorf("invalid catalog container max size: %d", maxSize) + } + if header.MaxSize != maxSize { + _ = f.Close() + return nil, fmt.Errorf("container max size mismatch: header=%d catalog=%d", header.MaxSize, maxSize) + } + if stat.Size() > header.MaxSize { + _ = f.Close() + return nil, fmt.Errorf("container size exceeds maximum: size=%d max_size=%d", stat.Size(), header.MaxSize) + } if !readonly { if _, err := f.Seek(stat.Size(), io.SeekStart); err != nil { _ = f.Close() @@ -151,8 +164,8 @@ func (c *FileContainer) Append(data []byte) (int64, error) { return 0, fmt.Errorf("container is read-only") } - if c.offset+int64(len(data)) > c.maxSize { - return 0, ErrContainerFull + if err := validateContainerRange("container append", c.offset, int64(len(data)), c.maxSize); err != nil { + return 0, fmt.Errorf("%w: %v", ErrContainerFull, err) } off := c.offset @@ -186,13 +199,19 @@ func (c *FileContainer) ReadAt(offset int64, size int64) ([]byte, error) { if c.f == nil { return nil, fmt.Errorf("container is closed") } + if err := validateContainerRange("container read", offset, size, c.Size()); err != nil { + return nil, err + } + if uint64(size) > uint64(^uint(0)>>1) { + return nil, fmt.Errorf("container read length exceeds platform int range: %d", size) + } if !c.readonly { if err := c.flushPending(); err != nil { return nil, err } } - buf := make([]byte, size) + buf := make([]byte, int(size)) n, err := c.f.ReadAt(buf, offset) if err != nil && err != io.EOF { @@ -207,6 +226,30 @@ func (c *FileContainer) ReadAt(offset int64, size int64) ([]byte, error) { return buf, nil } +// validateContainerRange checks that [offset, offset+length) is within limit +// without computing offset+length, which could overflow int64. +func validateContainerRange(label string, offset, length, limit int64) error { + if label == "" { + label = "container range" + } + if offset < 0 { + return fmt.Errorf("%s offset must be non-negative: %d", label, offset) + } + if length < 0 { + return fmt.Errorf("%s length must be non-negative: %d", label, length) + } + if limit < 0 { + return fmt.Errorf("%s limit must be non-negative: %d", label, limit) + } + if offset > limit { + return fmt.Errorf("%s offset exceeds limit: offset=%d limit=%d", label, offset, limit) + } + if length > limit-offset { + return fmt.Errorf("%s exceeds limit: offset=%d length=%d limit=%d", label, offset, length, limit) + } + return nil +} + func (c *FileContainer) Size() int64 { return c.offset } @@ -533,12 +576,15 @@ func GetOrCreateOpenContainerInDirExcluding(db db.DBTX, containersDir string, ex } func UpdateContainerSize(tx db.DBTX, containerID int64, newSize int64) error { - _, err := tx.Exec( + result, err := tx.Exec( `UPDATE container SET current_size = $1 WHERE id = $2`, newSize, containerID, ) - return err + if err != nil { + return err + } + return db.RequireExactlyOneRow(result, "update container size") } func SealContainer(tx db.DBTX, containerID int64, filename string) error { @@ -582,7 +628,7 @@ func sealContainerInDirWithFS(tx db.DBTX, containerID int64, filename string, co } // Update DB: mark sealed and clear the sealing-in-progress flag atomically. - _, err = tx.Exec(` + result, err := tx.Exec(` UPDATE container SET sealed = TRUE, sealing = FALSE, @@ -593,6 +639,9 @@ func sealContainerInDirWithFS(tx db.DBTX, containerID int64, filename string, co if err != nil { return fmt.Errorf("update/seal container failed: %w", err) } + if err := db.RequireExactlyOneRow(result, "seal container"); err != nil { + return fmt.Errorf("update/seal container failed: %w", err) + } return nil } @@ -610,14 +659,14 @@ func isQuarantineableContainer(dbconn *sql.DB, containerID int64) bool { // quarantine flag is set; any other stat error is returned as fatal. func quarantineContainerUpdateQueryAndArgs(backend db.Backend, containerID int64, info os.FileInfo, statErr error) (string, []any, error) { updateQ := `UPDATE container SET quarantine = TRUE, sealing = FALSE WHERE id = $1` - updateWithSizeQ := `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = $2, max_size = $2 WHERE id = $1` + updateWithSizeQ := `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = $2 WHERE id = $1` if backend == db.BackendSQLite { updateQ = `UPDATE container SET quarantine = TRUE, sealing = FALSE WHERE id = ?` - updateWithSizeQ = `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = ?, max_size = ? WHERE id = ?` + updateWithSizeQ = `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = ? WHERE id = ?` } if statErr == nil { if backend == db.BackendSQLite { - return updateWithSizeQ, []any{info.Size(), info.Size(), containerID}, nil + return updateWithSizeQ, []any{info.Size(), containerID}, nil } return updateWithSizeQ, []any{containerID, info.Size()}, nil } diff --git a/internal/container/container_seam_test.go b/internal/container/container_seam_test.go index eccb9102..225d58cf 100644 --- a/internal/container/container_seam_test.go +++ b/internal/container/container_seam_test.go @@ -28,6 +28,15 @@ func openSeamTestDB(t *testing.T) *sql.DB { return dbconn } +func seamCatalogContainerMaxSize(t *testing.T, dbconn *sql.DB, containerID int64) int64 { + t.Helper() + var maxSize int64 + if err := dbconn.QueryRow(`SELECT max_size FROM container WHERE id = ?`, containerID).Scan(&maxSize); err != nil { + t.Fatalf("load catalog container maximum: %v", err) + } + return maxSize +} + // TestContainerSeamDefaultFSPreservesWriteBehavior verifies that the default // OS-backed filesystem seam (LocalWriter.fs == fsx.Default()) writes bytes // correctly through the full container creation and append path. @@ -62,7 +71,7 @@ func TestContainerSeamDefaultFSPreservesWriteBehavior(t *testing.T) { // Verify bytes on disk at the reported offset. containerPath := filepath.Join(w.Dir(), placement.Filename) - rc, err := OpenReadOnlyContainer(containerPath, maxSize) + rc, err := OpenReadOnlyContainer(containerPath, seamCatalogContainerMaxSize(t, dbconn, placement.ContainerID)) if err != nil { t.Fatalf("open readonly container: %v", err) } @@ -116,7 +125,7 @@ func TestContainerSeamNoopFSMatchesDefaultBehavior(t *testing.T) { // Verify bytes are identical to what a default-FS run would produce. containerPath := filepath.Join(w.Dir(), placement.Filename) - rc, err := OpenReadOnlyContainer(containerPath, maxSize) + rc, err := OpenReadOnlyContainer(containerPath, seamCatalogContainerMaxSize(t, dbconn, placement.ContainerID)) if err != nil { t.Fatalf("open readonly container: %v", err) } @@ -204,7 +213,10 @@ func TestContainerFilesystemEquivalenceDefaultAndNoop(t *testing.T) { mustNoErr(t, txDefault.Commit(), "commit tx (default)") mustNoErr(t, wDefault.FinalizeContainer(), "finalize container (default)") - rcDefault, err := OpenReadOnlyContainer(filepath.Join(wDefault.Dir(), placementDefault.Filename), maxSize) + rcDefault, err := OpenReadOnlyContainer( + filepath.Join(wDefault.Dir(), placementDefault.Filename), + seamCatalogContainerMaxSize(t, dbDefault, placementDefault.ContainerID), + ) mustNoErr(t, err, "open readonly container (default)") defer func() { _ = rcDefault.Close() }() @@ -227,7 +239,10 @@ func TestContainerFilesystemEquivalenceDefaultAndNoop(t *testing.T) { mustNoErr(t, txNoop.Commit(), "commit tx (noop)") mustNoErr(t, wNoop.FinalizeContainer(), "finalize container (noop)") - rcNoop, err := OpenReadOnlyContainer(filepath.Join(wNoop.Dir(), placementNoop.Filename), maxSize) + rcNoop, err := OpenReadOnlyContainer( + filepath.Join(wNoop.Dir(), placementNoop.Filename), + seamCatalogContainerMaxSize(t, dbNoop, placementNoop.ContainerID), + ) mustNoErr(t, err, "open readonly container (noop)") defer func() { _ = rcNoop.Close() }() diff --git a/internal/container/container_test.go b/internal/container/container_test.go index b8d5f816..cd9494a8 100644 --- a/internal/container/container_test.go +++ b/internal/container/container_test.go @@ -3,12 +3,15 @@ package container import ( "database/sql" "errors" + "fmt" + "math" "os" "path/filepath" "strings" "testing" "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/fsx" _ "github.com/mattn/go-sqlite3" ) @@ -66,12 +69,28 @@ func TestFileContainerAppendFailsWhenFull(t *testing.T) { } } -func TestFileContainerReadAtFailsOnShortRead(t *testing.T) { - c := openWritableTestContainer(t, ContainerHdrLen+32) +func TestFileContainerReadAtFailsClosedWhenFileShrinksAfterOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "shrinking.bin") + createTestContainerFile(t, path, ContainerHdrLen+32) + c, err := OpenWritableContainer(path, ContainerHdrLen+32) + if err != nil { + t.Fatalf("open writable container: %v", err) + } defer func() { _ = c.Close() }() - // No payload has been written, so reads from the payload region are short. - _, err := c.ReadAt(ContainerHdrLen, 1) + if _, err := c.Append([]byte("x")); err != nil { + t.Fatalf("append payload: %v", err) + } + if err := c.Sync(); err != nil { + t.Fatalf("sync payload: %v", err) + } + if err := os.Truncate(path, ContainerHdrLen); err != nil { + t.Fatalf("truncate behind open container: %v", err) + } + + // The open handle still records the pre-truncation logical size, so the + // lower-level short-read check remains the fail-closed fallback. + _, err = c.ReadAt(ContainerHdrLen, 1) if err == nil || !strings.Contains(err.Error(), "short read") { t.Fatalf("expected short-read error contract, got: %v", err) } @@ -173,11 +192,146 @@ func TestFileContainerTruncateDiscardsPendingWritesWithoutSync(t *testing.T) { t.Fatalf("expected logical size reset to %d, got %d", ContainerHdrLen, got) } _, err = c.ReadAt(ContainerHdrLen, 1) - if err == nil || !strings.Contains(err.Error(), "short read") { + if err == nil || !strings.Contains(err.Error(), "exceeds limit") { t.Fatalf("expected no payload after truncate, got: %v", err) } } +func TestValidateContainerRangeBoundaries(t *testing.T) { + tests := []struct { + name string + offset, length int64 + limit int64 + wantErr bool + }{ + {name: "zero at start", offset: 0, length: 0, limit: 10}, + {name: "zero at end", offset: 10, length: 0, limit: 10}, + {name: "exact end", offset: 4, length: 6, limit: 10}, + {name: "negative offset", offset: -1, length: 1, limit: 10, wantErr: true}, + {name: "negative length", offset: 0, length: -1, limit: 10, wantErr: true}, + {name: "negative limit", offset: 0, length: 0, limit: -1, wantErr: true}, + {name: "offset past limit", offset: 11, length: 0, limit: 10, wantErr: true}, + {name: "length past limit", offset: 4, length: 7, limit: 10, wantErr: true}, + {name: "overflow shape", offset: math.MaxInt64 - 1, length: 2, limit: math.MaxInt64, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validateContainerRange("test range", tc.offset, tc.length, tc.limit) + if (err != nil) != tc.wantErr { + t.Fatalf("validateContainerRange(%d, %d, %d) error=%v wantErr=%v", tc.offset, tc.length, tc.limit, err, tc.wantErr) + } + }) + } +} + +func TestFileContainerReadAtRejectsInvalidRangeBeforeAllocation(t *testing.T) { + c := openWritableTestContainer(t, ContainerHdrLen+32) + defer func() { _ = c.Close() }() + + tests := []struct { + name string + offset, size int64 + }{ + {name: "negative offset", offset: -1, size: 1}, + {name: "negative size", offset: ContainerHdrLen, size: -1}, + {name: "offset past eof", offset: ContainerHdrLen + 1, size: 0}, + {name: "range past eof", offset: ContainerHdrLen, size: 1}, + {name: "huge size", offset: ContainerHdrLen, size: math.MaxInt64}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if _, err := c.ReadAt(tc.offset, tc.size); err == nil { + t.Fatalf("expected invalid range offset=%d size=%d to fail", tc.offset, tc.size) + } + }) + } +} + +func TestFileContainerReadAtAllowsZeroLengthAndExactEOF(t *testing.T) { + c := openWritableTestContainer(t, ContainerHdrLen+32) + defer func() { _ = c.Close() }() + + if _, err := c.Append([]byte("data")); err != nil { + t.Fatalf("append payload: %v", err) + } + if got, err := c.ReadAt(ContainerHdrLen+4, 0); err != nil || len(got) != 0 { + t.Fatalf("zero-length EOF read got=%v err=%v", got, err) + } + got, err := c.ReadAt(ContainerHdrLen, 4) + if err != nil || string(got) != "data" { + t.Fatalf("exact-EOF read got=%q err=%v", got, err) + } +} + +func TestOpenExistingContainerRejectsHeaderCatalogMaxSizeMismatch(t *testing.T) { + const headerMax = ContainerHdrLen + 128 + path := filepath.Join(t.TempDir(), "mismatch.bin") + createTestContainerFile(t, path, headerMax) + + for _, readonly := range []bool{true, false} { + _, err := openExistingContainer(readonly, path, headerMax+1, fsx.Default()) + if err == nil || !strings.Contains(err.Error(), "container max size mismatch") { + t.Fatalf("readonly=%v expected max-size mismatch, got %v", readonly, err) + } + } +} + +func TestOpenExistingContainerRejectsPhysicalSizeBeyondDeclaredMaximum(t *testing.T) { + const maxSize = ContainerHdrLen + 1 + path := filepath.Join(t.TempDir(), "oversized.bin") + createTestContainerFile(t, path, maxSize) + f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatalf("open container for append: %v", err) + } + if _, err := f.Write([]byte("xx")); err != nil { + _ = f.Close() + t.Fatalf("append oversized bytes: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("close oversized container: %v", err) + } + + _, err = OpenReadOnlyContainer(path, maxSize) + if err == nil || !strings.Contains(err.Error(), "container size exceeds maximum") { + t.Fatalf("expected oversized-container error, got %v", err) + } +} + +func TestOpenExistingContainerAcceptsSupportedMatchingHeaderMaxSize(t *testing.T) { + const maxSize = ContainerHdrLen + 128 + for _, major := range []uint16{LegacyContainerFormatVersionMajor, ContainerFormatVersionMajor} { + t.Run(fmt.Sprintf("major_%d", major), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "supported.bin") + writeContainerHeaderFixture(t, path, major, maxSize) + c, err := OpenReadOnlyContainer(path, maxSize) + if err != nil { + t.Fatalf("open supported header major=%d: %v", major, err) + } + if err := c.Close(); err != nil { + t.Fatalf("close supported container: %v", err) + } + }) + } +} + +func TestFileContainerAppendRejectsOverflowAsContainerFull(t *testing.T) { + path := filepath.Join(t.TempDir(), "append-overflow.bin") + createTestContainerFile(t, path, math.MaxInt64) + c, err := OpenWritableContainer(path, math.MaxInt64) + if err != nil { + t.Fatalf("open writable container: %v", err) + } + defer func() { _ = c.Close() }() + c.offset = math.MaxInt64 - 1 + + if _, err := c.Append([]byte("xx")); !errors.Is(err, ErrContainerFull) { + t.Fatalf("expected ErrContainerFull for overflowing append, got %v", err) + } +} + func TestFileContainerSyncSkipsRedundantFsyncWithoutNewWrites(t *testing.T) { path := filepath.Join(t.TempDir(), "redundant-sync.bin") createTestContainerFile(t, path, ContainerHdrLen+128) @@ -274,7 +428,7 @@ func TestBrokenOpenContainerErrorNilReceiverBehavior(t *testing.T) { } } -func TestQuarantineContainerInDirUpdatesSizesToPhysicalFile(t *testing.T) { +func TestQuarantineContainerInDirUpdatesCurrentSizeAndPreservesMaximum(t *testing.T) { dbconn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("open sqlite db: %v", err) @@ -341,8 +495,8 @@ func TestQuarantineContainerInDirUpdatesSizesToPhysicalFile(t *testing.T) { if currentSize != info.Size() { t.Fatalf("expected current_size=%d, got %d", info.Size(), currentSize) } - if maxSize != info.Size() { - t.Fatalf("expected max_size=%d, got %d", info.Size(), maxSize) + if maxSize != ContainerHdrLen+128 { + t.Fatalf("expected max_size=%d to remain unchanged, got %d", ContainerHdrLen+128, maxSize) } } diff --git a/internal/container/format.go b/internal/container/format.go index a3e14e3d..68568d6f 100644 --- a/internal/container/format.go +++ b/internal/container/format.go @@ -72,6 +72,10 @@ type Header struct { } func writeNewContainerHeader(f fsx.File, maxSize int64) error { + if maxSize <= ContainerHdrLen { + return fmt.Errorf("invalid container max size: %d", maxSize) + } + h := make([]byte, ContainerHdrLen) // 0..7 magic @@ -105,8 +109,14 @@ func writeNewContainerHeader(f fsx.File, maxSize int64) error { // 56..63 reserved (left as zero) - _, err := f.Write(h) - return err + n, err := f.Write(h) + if err != nil { + return err + } + if n != len(h) { + return io.ErrShortWrite + } + return nil } func readAndValidateContainerHeader(f fsx.File) (Header, error) { @@ -145,13 +155,18 @@ func readAndValidateContainerHeader(f fsx.File) (Header, error) { codecID = binary.LittleEndian.Uint16(h[hdrCodecID : hdrCodecID+2]) } + maxSize := int64(binary.LittleEndian.Uint64(h[hdrMaxSize:hdrUIDStart])) + if maxSize <= ContainerHdrLen { + return Header{}, fmt.Errorf("invalid container max size: %d", maxSize) + } + return Header{ FormatMajor: major, FormatMinor: minor, HeaderLen: hdrLen, Flags: binary.LittleEndian.Uint32(h[hdrFlags:hdrCreatedAt]), CreatedAt: int64(binary.LittleEndian.Uint64(h[hdrCreatedAt:hdrMaxSize])), - MaxSize: int64(binary.LittleEndian.Uint64(h[hdrMaxSize:hdrUIDStart])), + MaxSize: maxSize, CodecID: codecID, }, nil } diff --git a/internal/container/format_test.go b/internal/container/format_test.go index d3e6d49e..10a41567 100644 --- a/internal/container/format_test.go +++ b/internal/container/format_test.go @@ -2,12 +2,40 @@ package container import ( "encoding/binary" + "errors" + "fmt" "hash/crc32" + "io" "os" + "path/filepath" "strings" "testing" + + "github.com/franchoy/coldkeep/internal/fsx" ) +func writeContainerHeaderFixture(t *testing.T, path string, major uint16, maxSize int64) { + t.Helper() + hdr := make([]byte, ContainerHdrLen) + copy(hdr[hdrMagicStart:hdrMagicEnd], []byte(ContainerMagic)) + binary.LittleEndian.PutUint16(hdr[hdrVersionMajor:hdrVersionMinor], major) + binary.LittleEndian.PutUint16(hdr[hdrVersionMinor:hdrHeaderLen], 0) + binary.LittleEndian.PutUint32(hdr[hdrHeaderLen:hdrFlags], uint32(ContainerHdrLen)) + binary.LittleEndian.PutUint64(hdr[hdrMaxSize:hdrUIDStart], uint64(maxSize)) + binary.LittleEndian.PutUint32(hdr[hdrCRC:hdrCodecID], computeHeaderCRC(hdr, major)) + if err := os.WriteFile(path, hdr, 0o600); err != nil { + t.Fatalf("write container header fixture: %v", err) + } +} + +type shortWriteContainerFile struct { + fsx.File +} + +func (shortWriteContainerFile) Write(p []byte) (int, error) { + return len(p) - 1, nil +} + func TestWriteNewContainerHeader_UsesStableFormatVersionAndCodecHint(t *testing.T) { tmp, err := os.CreateTemp(t.TempDir(), "container-header-*.bin") if err != nil { @@ -175,3 +203,47 @@ func TestReadAndValidateContainerHeader_RejectsCRCMismatch(t *testing.T) { t.Fatalf("expected crc-mismatch error contract, got: %v", err) } } + +func TestReadAndValidateContainerHeaderRejectsInvalidMaxSizeAcrossSupportedVersions(t *testing.T) { + for _, major := range []uint16{LegacyContainerFormatVersionMajor, ContainerFormatVersionMajor} { + for _, maxSize := range []int64{-1, 0, ContainerHdrLen} { + t.Run(fmt.Sprintf("major_%d/max_%d", major, maxSize), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "invalid-max.bin") + writeContainerHeaderFixture(t, path, major, maxSize) + f, err := os.Open(path) + if err != nil { + t.Fatalf("open header fixture: %v", err) + } + defer func() { _ = f.Close() }() + + _, err = readAndValidateContainerHeader(f) + if err == nil || !strings.Contains(err.Error(), "invalid container max size") { + t.Fatalf("expected invalid max-size error, got %v", err) + } + }) + } + } +} + +func TestWriteNewContainerHeaderRejectsInvalidMaxSize(t *testing.T) { + for _, maxSize := range []int64{-1, 0, ContainerHdrLen} { + t.Run(fmt.Sprintf("max_%d", maxSize), func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "invalid-write-max-*.bin") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + defer func() { _ = f.Close() }() + + if err := writeNewContainerHeader(f, maxSize); err == nil || !strings.Contains(err.Error(), "invalid container max size") { + t.Fatalf("expected invalid max-size error, got %v", err) + } + }) + } +} + +func TestWriteNewContainerHeaderRejectsShortWrite(t *testing.T) { + err := writeNewContainerHeader(shortWriteContainerFile{}, ContainerHdrLen+1) + if !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("expected io.ErrShortWrite, got %v", err) + } +} diff --git a/internal/container/local_writer.go b/internal/container/local_writer.go index bb0dcb09..47dc92d4 100644 --- a/internal/container/local_writer.go +++ b/internal/container/local_writer.go @@ -143,7 +143,11 @@ func (w *LocalWriter) AppendPayload(tx db.DBTX, payload []byte) (LocalPlacement, previousSize = w.activeSize // Mark sealing inside the transaction that already owns the container row. - if _, err := tx.Exec(`UPDATE container SET sealing = TRUE WHERE id = $1`, previousID); err != nil { + result, err := tx.Exec(`UPDATE container SET sealing = TRUE WHERE id = $1`, previousID) + if err != nil { + return LocalPlacement{}, fmt.Errorf("mark rotation container %d sealing: %w", previousID, err) + } + if err := db.RequireExactlyOneRow(result, "mark rotation container sealing"); err != nil { return LocalPlacement{}, fmt.Errorf("mark rotation container %d sealing: %w", previousID, err) } diff --git a/internal/container/mutation_cardinality_test.go b/internal/container/mutation_cardinality_test.go new file mode 100644 index 00000000..333278c3 --- /dev/null +++ b/internal/container/mutation_cardinality_test.go @@ -0,0 +1,121 @@ +package container + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/franchoy/coldkeep/internal/db" +) + +func TestRequiredContainerMutationsFailClosedOnMissingRows(t *testing.T) { + dbconn := setupContainerOpsTestDB(t) + defer func() { _ = dbconn.Close() }() + + t.Run("update-size", func(t *testing.T) { + err := UpdateContainerSize(dbconn, 404, ContainerHdrLen) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) + + t.Run("simulated-seal", func(t *testing.T) { + err := NewSimulatedWriter(ContainerHdrLen+128).SealContainer(dbconn, 404, "", "") + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) +} + +func TestSealContainerFailsClosedWhenUpdateMatchesZero(t *testing.T) { + dbconn := setupContainerOpsTestDB(t) + defer func() { _ = dbconn.Close() }() + + dir := t.TempDir() + filename := "phase17-seal.bin" + maxSize := int64(ContainerHdrLen + 128) + createTestContainerFile(t, filepath.Join(dir, filename), maxSize) + + result, err := dbconn.Exec( + `INSERT INTO container (filename, current_size, max_size, sealed, sealing, quarantine) + VALUES (?, ?, ?, FALSE, TRUE, FALSE)`, + filename, + int64(ContainerHdrLen), + maxSize, + ) + if err != nil { + t.Fatalf("insert container row: %v", err) + } + containerID, err := result.LastInsertId() + if err != nil { + t.Fatalf("container id: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_container_seal + BEFORE UPDATE OF sealed ON container + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create ignored-seal trigger: %v", err) + } + + tx, err := dbconn.Begin() + if err != nil { + t.Fatalf("begin seal transaction: %v", err) + } + err = SealContainerInDir(tx, containerID, filename, dir) + if !errors.Is(err, db.ErrMutationCardinality) { + _ = tx.Rollback() + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback seal transaction: %v", err) + } + + var sealed, sealing bool + if err := dbconn.QueryRow(`SELECT sealed, sealing FROM container WHERE id = ?`, containerID).Scan(&sealed, &sealing); err != nil { + t.Fatalf("read container state: %v", err) + } + if sealed || !sealing { + t.Fatalf("unexpected state after rollback: sealed=%t sealing=%t", sealed, sealing) + } +} + +func TestLocalWriterRotationFailsClosedWhenSealingMarkerMatchesZero(t *testing.T) { + const containerID = int64(1) + const maxSize = int64(ContainerHdrLen + 12) + dbconn := openContainerTestDB(t, containerID, ContainerHdrLen+10, maxSize) + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_rotation_sealing + BEFORE UPDATE OF sealing ON container + WHEN OLD.id = 1 + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create ignored-rotation trigger: %v", err) + } + + tx, err := dbconn.Begin() + if err != nil { + t.Fatalf("begin rotation transaction: %v", err) + } + defer func() { _ = tx.Rollback() }() + + handle := &fakeContainer{size: ContainerHdrLen + 10} + w := NewLocalWriterWithDirAndDB(t.TempDir(), maxSize, dbconn) + w.hasActive = true + w.activeID = containerID + w.activeFile = "c.bin" + w.activeHandle = handle + w.activeSize = ContainerHdrLen + 10 + + _, err = w.AppendPayload(tx, []byte("abc")) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if got := handle.Size(); got != ContainerHdrLen+10 { + t.Fatalf("rotation wrote bytes after missed sealing marker: size=%d", got) + } +} diff --git a/internal/container/payload.go b/internal/container/payload.go index 78a9df29..2a6d45f8 100644 --- a/internal/container/payload.go +++ b/internal/container/payload.go @@ -6,6 +6,15 @@ func ReadPayloadAt(c Container, offset int64, size int64) ([]byte, error) { if size < 0 { return nil, fmt.Errorf("invalid payload size: %d", size) } + if c == nil { + return nil, fmt.Errorf("container is nil") + } + if offset < ContainerHdrLen { + return nil, fmt.Errorf("invalid payload offset before container header: %d", offset) + } + if err := validateContainerRange("payload read", offset, size, c.Size()); err != nil { + return nil, err + } payload, err := c.ReadAt(offset, size) if err != nil { diff --git a/internal/container/payload_test.go b/internal/container/payload_test.go index 6ef775e4..351ffdfe 100644 --- a/internal/container/payload_test.go +++ b/internal/container/payload_test.go @@ -7,11 +7,14 @@ import ( ) // errContainer is a minimal Container stub whose ReadAt always returns a fixed error. -type errContainer struct{ err error } +type errContainer struct { + err error + size int64 +} func (e errContainer) Append(_ []byte) (int64, error) { return 0, nil } func (e errContainer) ReadAt(_ int64, _ int64) ([]byte, error) { return nil, e.err } -func (e errContainer) Size() int64 { return 0 } +func (e errContainer) Size() int64 { return e.size } func (e errContainer) Truncate(_ int64) error { return nil } func (e errContainer) Sync() error { return nil } func (e errContainer) Close() error { return nil } @@ -26,7 +29,7 @@ func TestReadPayloadAtFailsOnNegativeSize(t *testing.T) { func TestReadPayloadAtWrapsReadError(t *testing.T) { readErr := errors.New("disk I/O failure") - c := errContainer{err: readErr} + c := errContainer{err: readErr, size: ContainerHdrLen + 16} _, err := ReadPayloadAt(c, 64, 16) if err == nil || !strings.Contains(err.Error(), "read payload at offset") || @@ -48,12 +51,66 @@ func TestReadPayloadAtReturnsPayloadOnSuccess(t *testing.T) { } } +type trackingContainer struct { + size int64 + readCalls int +} + +func (c *trackingContainer) Append(_ []byte) (int64, error) { return 0, nil } +func (c *trackingContainer) ReadAt(_ int64, size int64) ([]byte, error) { + c.readCalls++ + return make([]byte, int(size)), nil +} +func (c *trackingContainer) Size() int64 { return c.size } +func (c *trackingContainer) Truncate(_ int64) error { return nil } +func (c *trackingContainer) Sync() error { return nil } +func (c *trackingContainer) Close() error { return nil } + +func TestReadPayloadAtRejectsInvalidRangeBeforeRead(t *testing.T) { + tests := []struct { + name string + offset int64 + size int64 + }{ + {name: "negative offset", offset: -1, size: 1}, + {name: "header overlap", offset: ContainerHdrLen - 1, size: 1}, + {name: "past EOF", offset: ContainerHdrLen + 5, size: 1}, + {name: "overflow shape", offset: ContainerHdrLen, size: int64(^uint64(0) >> 1)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &trackingContainer{size: ContainerHdrLen + 4} + if _, err := ReadPayloadAt(c, tc.offset, tc.size); err == nil { + t.Fatal("expected invalid payload range error") + } + if c.readCalls != 0 { + t.Fatalf("expected invalid range to be rejected before ReadAt, got %d calls", c.readCalls) + } + }) + } +} + +func TestReadPayloadAtAllowsFirstPayloadAndExactEOF(t *testing.T) { + c := &trackingContainer{size: ContainerHdrLen + 4} + + if got, err := ReadPayloadAt(c, ContainerHdrLen, 4); err != nil || len(got) != 4 { + t.Fatalf("read first payload through EOF: len=%d err=%v", len(got), err) + } + if got, err := ReadPayloadAt(c, c.Size(), 0); err != nil || len(got) != 0 { + t.Fatalf("read zero bytes at EOF: len=%d err=%v", len(got), err) + } + if c.readCalls != 2 { + t.Fatalf("expected two delegated reads, got %d", c.readCalls) + } +} + // okContainer returns a fixed byte slice from ReadAt regardless of offset/size. type okContainer struct{ data []byte } func (o okContainer) Append(_ []byte) (int64, error) { return 0, nil } func (o okContainer) ReadAt(_ int64, _ int64) ([]byte, error) { return o.data, nil } -func (o okContainer) Size() int64 { return int64(len(o.data)) } +func (o okContainer) Size() int64 { return ContainerHdrLen + int64(len(o.data)) } func (o okContainer) Truncate(_ int64) error { return nil } func (o okContainer) Sync() error { return nil } func (o okContainer) Close() error { return nil } diff --git a/internal/container/row_lock_backend_contract_test.go b/internal/container/row_lock_backend_contract_test.go new file mode 100644 index 00000000..c620569d --- /dev/null +++ b/internal/container/row_lock_backend_contract_test.go @@ -0,0 +1,292 @@ +package container + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +const phase10ContainerOperationTimeout = 5 * time.Second + +func TestContainerRowLockIntegrationAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + ctx, cancel := context.WithTimeout(context.Background(), phase10ContainerOperationTimeout) + defer cancel() + + seedPhase10ContainerRows(t, ctx, backend.DB) + + if backend.Kind == db.BackendSQLite { + testPhase10SQLiteContainerLockBoundary(t, ctx, backend.DB) + assertPhase10ContainerRowsReusable(t, ctx, backend.DB) + return + } + + t.Run("nowait_savepoint_recovery", func(t *testing.T) { + testPhase10PostgresContainerNowait(t, ctx, backend.DB) + }) + t.Run("skip_locked_candidate_order", func(t *testing.T) { + testPhase10PostgresContainerSkipLocked(t, ctx, backend.DB) + }) + + assertPhase10ContainerRowsReusable(t, ctx, backend.DB) + }) +} + +func seedPhase10ContainerRows(t *testing.T, ctx context.Context, database *sql.DB) { + t.Helper() + + _, err := database.ExecContext(ctx, ` + INSERT INTO container ( + id, filename, current_size, max_size, sealed, sealing, quarantine + ) VALUES + (1001, 'phase10-container-1.bin', 64, 4096, FALSE, FALSE, FALSE), + (1002, 'phase10-container-2.bin', 64, 4096, FALSE, FALSE, FALSE) + `) + if err != nil { + t.Fatalf("seed Phase 10 container rows: %v", err) + } +} + +func testPhase10SQLiteContainerLockBoundary( + t *testing.T, + ctx context.Context, + database *sql.DB, +) { + t.Helper() + + tx, err := database.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin SQLite container transaction: %v", err) + } + defer rollbackPhase10ContainerTx(t, tx) + + if err := lockContainerRowNowaitWithRetry(tx, database, 1001, 1, time.Millisecond); err != nil { + t.Fatalf("SQLite lock helper should execute its clause-free lookup: %v", err) + } + + id, _, _, err := selectOpenContainerExcluding(tx, database, 0) + if err != nil { + t.Fatalf("select first SQLite open container: %v", err) + } + if id != 1001 { + t.Fatalf("SQLite ordered lookup returned container %d, want 1001", id) + } + id, _, _, err = selectOpenContainerExcluding(tx, database, 1001) + if err != nil { + t.Fatalf("select SQLite open container excluding 1001: %v", err) + } + if id != 1002 { + t.Fatalf("SQLite exclusion lookup returned container %d, want 1002", id) + } + + if err := tx.Rollback(); err != nil { + t.Fatalf("roll back SQLite container transaction: %v", err) + } +} + +func testPhase10PostgresContainerNowait( + t *testing.T, + ctx context.Context, + database *sql.DB, +) { + t.Helper() + + connA := phase10ContainerConn(t, ctx, database) + defer closePhase10ContainerConn(t, connA, "NOWAIT locker") + connB := phase10ContainerConn(t, ctx, database) + defer closePhase10ContainerConn(t, connB, "NOWAIT contender") + + txA, err := connA.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin PostgreSQL container lock holder: %v", err) + } + defer rollbackPhase10ContainerTx(t, txA) + + lockQuery := db.QueryWithOptionalForUpdate(database, "SELECT id FROM container WHERE id = $1") + var lockedID int64 + if err := txA.QueryRowContext(ctx, lockQuery, 1001).Scan(&lockedID); err != nil { + t.Fatalf("lock PostgreSQL container row: %v", err) + } + if lockedID != 1001 { + t.Fatalf("NOWAIT locker selected container %d, want 1001", lockedID) + } + + txB, err := connB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin PostgreSQL NOWAIT contender: %v", err) + } + defer rollbackPhase10ContainerTx(t, txB) + + err = lockContainerRowNowaitWithRetry(txB, database, 1001, 1, time.Millisecond) + if !errors.Is(err, ErrContainerLockContention) { + t.Fatalf("NOWAIT helper error = %v, want ErrContainerLockContention", err) + } + + var one int + if err := txB.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { + t.Fatalf("NOWAIT savepoint should leave transaction usable: %v", err) + } + if one != 1 { + t.Fatalf("transaction reuse query returned %d, want 1", one) + } + + if err := txB.Rollback(); err != nil { + t.Fatalf("roll back PostgreSQL NOWAIT contender: %v", err) + } + if err := txA.Commit(); err != nil { + t.Fatalf("commit PostgreSQL container lock holder: %v", err) + } + + txAfter, err := connB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin PostgreSQL post-release transaction: %v", err) + } + defer rollbackPhase10ContainerTx(t, txAfter) + + if err := lockContainerRowNowaitWithRetry( + txAfter, + database, + 1001, + 1, + time.Millisecond, + ); err != nil { + t.Fatalf("NOWAIT helper after release: %v", err) + } + if err := txAfter.Rollback(); err != nil { + t.Fatalf("roll back PostgreSQL post-release transaction: %v", err) + } +} + +func testPhase10PostgresContainerSkipLocked( + t *testing.T, + ctx context.Context, + database *sql.DB, +) { + t.Helper() + + connA := phase10ContainerConn(t, ctx, database) + defer closePhase10ContainerConn(t, connA, "SKIP LOCKED locker") + connB := phase10ContainerConn(t, ctx, database) + defer closePhase10ContainerConn(t, connB, "SKIP LOCKED selector") + + txA, err := connA.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin PostgreSQL SKIP LOCKED holder: %v", err) + } + defer rollbackPhase10ContainerTx(t, txA) + + lockQuery := db.QueryWithOptionalForUpdate(database, "SELECT id FROM container WHERE id = $1") + var lockedID int64 + if err := txA.QueryRowContext(ctx, lockQuery, 1001).Scan(&lockedID); err != nil { + t.Fatalf("lock lower PostgreSQL container row: %v", err) + } + if lockedID != 1001 { + t.Fatalf("SKIP LOCKED locker selected container %d, want 1001", lockedID) + } + + txB, err := connB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin PostgreSQL SKIP LOCKED selector: %v", err) + } + defer rollbackPhase10ContainerTx(t, txB) + + selectedID, _, _, err := selectOpenContainerExcluding(txB, database, 0) + if err != nil { + t.Fatalf("select unlocked PostgreSQL container: %v", err) + } + if selectedID != 1002 { + t.Fatalf("SKIP LOCKED selected container %d, want 1002", selectedID) + } + + if err := txB.Rollback(); err != nil { + t.Fatalf("roll back PostgreSQL SKIP LOCKED selector: %v", err) + } + if err := txA.Rollback(); err != nil { + t.Fatalf("release lower PostgreSQL container row: %v", err) + } + + txAfter, err := connB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin PostgreSQL post-release selector: %v", err) + } + defer rollbackPhase10ContainerTx(t, txAfter) + + selectedID, _, _, err = selectOpenContainerExcluding(txAfter, database, 0) + if err != nil { + t.Fatalf("select PostgreSQL container after release: %v", err) + } + if selectedID != 1001 { + t.Fatalf("post-release selector returned container %d, want 1001", selectedID) + } + selectedID, _, _, err = selectOpenContainerExcluding(txAfter, database, 1001) + if err != nil { + t.Fatalf("select PostgreSQL container excluding 1001: %v", err) + } + if selectedID != 1002 { + t.Fatalf("PostgreSQL exclusion selector returned container %d, want 1002", selectedID) + } + if err := txAfter.Rollback(); err != nil { + t.Fatalf("roll back PostgreSQL post-release selector: %v", err) + } +} + +func phase10ContainerConn(t *testing.T, ctx context.Context, database *sql.DB) *sql.Conn { + t.Helper() + + conn, err := database.Conn(ctx) + if err != nil { + t.Fatalf("reserve PostgreSQL container connection: %v", err) + } + return conn +} + +func closePhase10ContainerConn(t *testing.T, conn *sql.Conn, role string) { + t.Helper() + + if err := conn.Close(); err != nil { + t.Errorf("close PostgreSQL container %s connection: %v", role, err) + } +} + +func rollbackPhase10ContainerTx(t *testing.T, tx *sql.Tx) { + t.Helper() + + if tx == nil { + return + } + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + t.Errorf("clean up Phase 10 container transaction: %v", err) + } +} + +func assertPhase10ContainerRowsReusable( + t *testing.T, + ctx context.Context, + database *sql.DB, +) { + t.Helper() + + var one int + if err := database.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { + t.Fatalf("final container connection reuse query: %v", err) + } + if one != 1 { + t.Fatalf("final container reuse query returned %d, want 1", one) + } + + var count int + if err := database.QueryRowContext( + ctx, + "SELECT COUNT(*) FROM container WHERE id IN (1001, 1002)", + ).Scan(&count); err != nil { + t.Fatalf("verify Phase 10 container fixture rows: %v", err) + } + if count != 2 { + t.Fatalf("Phase 10 container row count = %d, want 2", count) + } +} diff --git a/internal/container/simulated_writer.go b/internal/container/simulated_writer.go index 4b9b4479..8fae107a 100644 --- a/internal/container/simulated_writer.go +++ b/internal/container/simulated_writer.go @@ -130,12 +130,15 @@ func (w *SimulatedWriter) AppendPayload(tx db.DBTX, payload []byte) (LocalPlacem } func (w *SimulatedWriter) SealContainer(tx db.DBTX, containerID int64, _ string, _ string) error { - _, err := tx.Exec( + result, err := tx.Exec( `UPDATE container SET sealed = TRUE, container_hash = $1 WHERE id = $2`, "SIMULATED", containerID, ) - return err + if err != nil { + return err + } + return db.RequireExactlyOneRow(result, "seal simulated container") } func (w *SimulatedWriter) ensureActive(tx db.DBTX) error { diff --git a/internal/coordination/contract.go b/internal/coordination/contract.go new file mode 100644 index 00000000..a0105ec3 --- /dev/null +++ b/internal/coordination/contract.go @@ -0,0 +1,195 @@ +// Package coordination defines Coldkeep's same-host, local-filesystem +// repository coordination contract. +// +// A direct library caller operating on a real or shared repository must +// acquire a repository Lease before opening the database, initializing its +// schema, or constructing an Engine. The caller must hold that Lease through +// the repository operation, operation cleanup, and database/runtime cleanup, +// then release it afterward. The ordering is repository Lease, database/schema, +// database locks and transactions, then filesystem work. +// +// Coldkeep intentionally does not acquire a Lease inside individual Engine +// methods. Direct Engine calls remain appropriate for isolated temporary +// repositories and single-owner test fixtures that do not claim repository +// coordination evidence. A direct Engine call against a shared real repository +// is not safe-concurrency proof without an outer Lease. +// +// Coordination is repository-wide, exclusive, fail-fast, and non-reentrant. +// It is not distributed coordination. In particular, using the same database +// with different container namespaces is unsupported and is not made safe by +// this Lease. +package coordination + +import ( + "context" + "errors" + "fmt" +) + +// Mode identifies a repository coordination compatibility mode. +type Mode string + +const ( + // ModeExclusive is the only v1.13.11 coordination mode. Every participating + // repository operation conflicts with every other participating operation. + ModeExclusive Mode = "exclusive" +) + +// Operation is a stable, path-free identifier used for policy and diagnostics. +type Operation string + +const ( + OperationStore Operation = "store" + OperationStoreFolder Operation = "store-folder" + OperationRestore Operation = "restore" + OperationRemove Operation = "remove" + OperationRepair Operation = "repair" + OperationGarbageCollect Operation = "gc" + OperationStats Operation = "stats" + OperationInspect Operation = "inspect" + OperationList Operation = "list" + OperationSearch Operation = "search" + OperationVerify Operation = "verify" + OperationDoctor Operation = "doctor" + OperationConfigGet Operation = "config-get" + OperationConfigSet Operation = "config-set" + OperationSnapshotCreate Operation = "snapshot-create" + OperationSnapshotDelete Operation = "snapshot-delete" + OperationSnapshotRestore Operation = "snapshot-restore" + OperationSnapshotList Operation = "snapshot-list" + OperationSnapshotShow Operation = "snapshot-show" + OperationSnapshotStats Operation = "snapshot-stats" + OperationSnapshotDiff Operation = "snapshot-diff" + OperationStartupRecovery Operation = "startup-recovery" + OperationSchemaBootstrap Operation = "schema-bootstrap" + OperationSchemaMigration Operation = "schema-migration" +) + +// Request describes one fail-fast exclusive acquisition. +type Request struct { + Operation Operation + Mode Mode + Owner Owner +} + +// Lease is the explicit ownership token returned by a Coordinator. +// +// Release implementations must be idempotent. Finalizers are prohibited; +// callers retain explicit lifecycle ownership. +type Lease interface { + Release() error +} + +// Coordinator acquires one repository-wide lease. +// +// Phase 12 implementations must be fail-fast and non-reentrant. A second +// acquisition of the same identity while held must return +// ErrNestedRepositoryAcquisition rather than reference-counting ownership. +type Coordinator interface { + Acquire(context.Context, Identity, Request) (Lease, error) +} + +// WithLease runs fn while holding a lease and centralizes the required release +// and error-combination lifecycle. It does not implement lock acquisition. +func WithLease( + ctx context.Context, + coordinator Coordinator, + identity Identity, + request Request, + fn func() error, +) (err error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + if coordinator == nil { + return fmt.Errorf("coordination: coordinator is required") + } + if fn == nil { + return fmt.Errorf("coordination: operation callback is required") + } + if err := ValidateRequest(identity, request); err != nil { + return err + } + + lease, err := coordinator.Acquire(ctx, identity, request) + if err != nil { + return err + } + if lease == nil { + return fmt.Errorf("coordination: coordinator returned a nil lease") + } + + defer func() { + releaseErr := lease.Release() + switch { + case err != nil && releaseErr != nil: + err = errors.Join(err, releaseErr) + case releaseErr != nil: + err = releaseErr + } + }() + + return fn() +} + +// ValidateRequest freezes the relationship between identity, operation, mode, +// and diagnostic owner metadata before native acquisition. +func ValidateRequest(identity Identity, request Request) error { + if err := ValidateIdentity(identity); err != nil { + return err + } + if request.Mode != ModeExclusive { + return fmt.Errorf("%w: unsupported mode %q", ErrRepositoryLockUnsupported, request.Mode) + } + if !isCanonicalOperation(request.Operation) { + return fmt.Errorf("coordination: unsupported operation %q", request.Operation) + } + if err := ValidateOwner(request.Owner); err != nil { + return err + } + if request.Owner.Operation != request.Operation { + return fmt.Errorf("coordination: owner operation %q does not match request %q", request.Owner.Operation, request.Operation) + } + if request.Owner.Mode != request.Mode { + return fmt.Errorf("coordination: owner mode %q does not match request %q", request.Owner.Mode, request.Mode) + } + if request.Owner.IdentityHash != identity.Hash { + return fmt.Errorf("coordination: owner identity does not match request identity") + } + return nil +} + +func isCanonicalOperation(operation Operation) bool { + switch operation { + case OperationStore, + OperationStoreFolder, + OperationRestore, + OperationRemove, + OperationRepair, + OperationGarbageCollect, + OperationStats, + OperationInspect, + OperationList, + OperationSearch, + OperationVerify, + OperationDoctor, + OperationConfigGet, + OperationConfigSet, + OperationSnapshotCreate, + OperationSnapshotDelete, + OperationSnapshotRestore, + OperationSnapshotList, + OperationSnapshotShow, + OperationSnapshotStats, + OperationSnapshotDiff, + OperationStartupRecovery, + OperationSchemaBootstrap, + OperationSchemaMigration: + return true + default: + return false + } +} diff --git a/internal/coordination/contract_test.go b/internal/coordination/contract_test.go new file mode 100644 index 00000000..96813d71 --- /dev/null +++ b/internal/coordination/contract_test.go @@ -0,0 +1,343 @@ +package coordination + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +var testOwnerStart = time.Date(2026, time.July, 25, 12, 0, 0, 0, time.UTC) + +type fakeLease struct { + releaseErr error + releaseCalls int + released bool + onRelease func() +} + +func (l *fakeLease) Release() error { + l.releaseCalls++ + if l.released { + return nil + } + if l.releaseErr == nil { + l.released = true + } + if l.onRelease != nil { + l.onRelease() + l.onRelease = nil + } + return l.releaseErr +} + +type fakeCoordinator struct { + lease Lease + acquireErr error + acquireCalls int + acquired bool +} + +func (c *fakeCoordinator) Acquire(context.Context, Identity, Request) (Lease, error) { + c.acquireCalls++ + if c.acquireErr != nil { + return nil, c.acquireErr + } + c.acquired = true + return c.lease, nil +} + +func TestWithLeaseLifecycle(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + lease := &fakeLease{} + coordinator := &fakeCoordinator{lease: lease} + callbackCalled := false + + err := WithLease(context.Background(), coordinator, identity, mustRequest(t, identity, OperationStore), func() error { + callbackCalled = true + if !coordinator.acquired { + t.Fatal("callback ran before acquisition") + } + if lease.releaseCalls != 0 { + t.Fatal("lease released before callback completed") + } + return nil + }) + if err != nil { + t.Fatalf("WithLease: %v", err) + } + if !callbackCalled || coordinator.acquireCalls != 1 || lease.releaseCalls != 1 { + t.Fatalf("unexpected lifecycle callback=%v acquire=%d release=%d", callbackCalled, coordinator.acquireCalls, lease.releaseCalls) + } +} + +func TestWithLeaseDoesNotRunAfterAcquireFailure(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + coordinator := &fakeCoordinator{acquireErr: ErrRepositoryBusy} + callbackCalled := false + + err := WithLease(context.Background(), coordinator, identity, mustRequest(t, identity, OperationVerify), func() error { + callbackCalled = true + return nil + }) + if !errors.Is(err, ErrRepositoryBusy) { + t.Fatalf("expected busy classification, got %v", err) + } + if callbackCalled { + t.Fatal("callback ran after acquisition failure") + } +} + +func TestWithLeaseCombinesOperationAndReleaseErrors(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + operationErr := errors.New("operation failed") + releaseErr := errors.New("release failed") + lease := &fakeLease{releaseErr: releaseErr} + + err := WithLease(context.Background(), &fakeCoordinator{lease: lease}, identity, mustRequest(t, identity, OperationRestore), func() error { + return operationErr + }) + if !errors.Is(err, operationErr) || !errors.Is(err, releaseErr) { + t.Fatalf("expected joined operation and release errors, got %v", err) + } + if !strings.HasPrefix(err.Error(), operationErr.Error()+"\n") { + t.Fatalf("operation error was not first in joined error: %v", err) + } + if lease.releaseCalls != 1 { + t.Fatalf("expected one release, got %d", lease.releaseCalls) + } +} + +func TestWithLeaseReturnsOperationErrorAfterReleaseSuccess(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + operationErr := errors.New("operation failed") + lease := &fakeLease{} + + err := WithLease(context.Background(), &fakeCoordinator{lease: lease}, identity, mustRequest(t, identity, OperationRemove), func() error { + return operationErr + }) + if !errors.Is(err, operationErr) { + t.Fatalf("expected operation error, got %v", err) + } + if lease.releaseCalls != 1 { + t.Fatalf("expected one release, got %d", lease.releaseCalls) + } +} + +func TestWithLeaseReturnsReleaseErrorAfterOperationSuccess(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + releaseErr := errors.New("release failed") + err := WithLease(context.Background(), &fakeCoordinator{ + lease: &fakeLease{releaseErr: releaseErr}, + }, identity, mustRequest(t, identity, OperationGarbageCollect), func() error { + return nil + }) + if !errors.Is(err, releaseErr) { + t.Fatalf("expected release error, got %v", err) + } +} + +func TestLeaseContractReleaseIsIdempotentAfterSuccess(t *testing.T) { + cleanupCalls := 0 + lease := &fakeLease{onRelease: func() { + cleanupCalls++ + }} + + if err := lease.Release(); err != nil { + t.Fatalf("first release: %v", err) + } + if err := lease.Release(); err != nil { + t.Fatalf("second release: %v", err) + } + if cleanupCalls != 1 { + t.Fatalf("cleanup calls=%d want=1", cleanupCalls) + } + if lease.releaseCalls != 2 { + t.Fatalf("release calls=%d want=2", lease.releaseCalls) + } +} + +func TestWithLeasePreservesContextCancellation(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + coordinator := &fakeCoordinator{lease: &fakeLease{}} + + err := WithLease(ctx, coordinator, identity, mustRequest(t, identity, OperationList), func() error { + t.Fatal("callback must not run for a cancelled context") + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context cancellation, got %v", err) + } + if coordinator.acquireCalls != 0 { + t.Fatalf("cancelled context reached coordinator %d times", coordinator.acquireCalls) + } +} + +func TestWithLeasePreservesExpiredDeadline(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + ctx, cancel := context.WithDeadline(context.Background(), time.Unix(0, 0)) + defer cancel() + coordinator := &fakeCoordinator{lease: &fakeLease{}} + + err := WithLease(ctx, coordinator, identity, mustRequest(t, identity, OperationInspect), func() error { + t.Fatal("callback must not run for an expired context") + return nil + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline classification, got %v", err) + } + if coordinator.acquireCalls != 0 { + t.Fatalf("expired context reached coordinator %d times", coordinator.acquireCalls) + } +} + +func TestWithLeaseRejectsUnsupportedModeAndInvalidInputs(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + coordinator := &fakeCoordinator{lease: &fakeLease{}} + + request := mustRequest(t, identity, OperationStats) + request.Mode = Mode("shared") + err := WithLease(context.Background(), coordinator, identity, request, func() error { return nil }) + if !errors.Is(err, ErrRepositoryLockUnsupported) { + t.Fatalf("expected unsupported classification, got %v", err) + } + + if err := WithLease(context.Background(), nil, identity, mustRequest(t, identity, OperationStats), func() error { return nil }); err == nil { + t.Fatal("expected nil coordinator error") + } +} + +func TestValidateRequestRejectsMismatchedOwner(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + request := mustRequest(t, identity, OperationStore) + request.Owner.Operation = OperationRemove + if err := ValidateRequest(identity, request); err == nil { + t.Fatal("expected owner operation mismatch") + } + + request = mustRequest(t, identity, OperationStore) + request.Owner.IdentityHash = strings.Repeat("0", sha256HexLength) + if err := ValidateRequest(identity, request); err == nil { + t.Fatal("expected owner identity mismatch") + } +} + +func TestCoordinationValidateRequestRejectsBlankAndUnknownOperations(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + for _, operation := range []Operation{"", "unknown"} { + request := mustRequest(t, identity, OperationStore) + request.Operation = operation + request.Owner.Operation = operation + if err := ValidateRequest(identity, request); err == nil { + t.Fatalf("expected operation %q to fail", operation) + } + } +} + +type nonReentrantFakeCoordinator struct { + mu sync.Mutex + held map[string]bool +} + +func (c *nonReentrantFakeCoordinator) Acquire(_ context.Context, identity Identity, _ Request) (Lease, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + c.held = make(map[string]bool) + } + if c.held[identity.Hash] { + return nil, fmt.Errorf("%w: %s", ErrNestedRepositoryAcquisition, identity.Hash) + } + c.held[identity.Hash] = true + return &fakeLease{onRelease: func() { + c.mu.Lock() + delete(c.held, identity.Hash) + c.mu.Unlock() + }}, nil +} + +func TestCoordinatorContractRejectsNestedIdentityAndSeparatesRepositories(t *testing.T) { + coordinator := &nonReentrantFakeCoordinator{} + firstIdentity := mustIdentity(t, t.TempDir()) + secondIdentity := mustIdentity(t, t.TempDir()) + request := mustRequest(t, firstIdentity, OperationStore) + + first, err := coordinator.Acquire(context.Background(), firstIdentity, request) + if err != nil { + t.Fatalf("acquire first identity: %v", err) + } + if _, err := coordinator.Acquire(context.Background(), firstIdentity, request); !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("expected nested acquisition error, got %v", err) + } + secondRequest := mustRequest(t, secondIdentity, OperationStore) + second, err := coordinator.Acquire(context.Background(), secondIdentity, secondRequest) + if err != nil { + t.Fatalf("independent repository should not contend: %v", err) + } + if err := second.Release(); err != nil { + t.Fatalf("release second identity: %v", err) + } + if err := first.Release(); err != nil { + t.Fatalf("release first identity: %v", err) + } + if _, err := coordinator.Acquire(context.Background(), firstIdentity, request); err != nil { + t.Fatalf("reacquire after release: %v", err) + } +} + +func TestCoordinatorContractTreatsAliasAsNestedIdentity(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "repository") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatalf("create target: %v", err) + } + alias := filepath.Join(root, "alias") + if err := os.Symlink(target, alias); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + direct := mustIdentity(t, target) + throughAlias := mustIdentity(t, alias) + coordinator := &nonReentrantFakeCoordinator{} + request := mustRequest(t, direct, OperationStore) + lease, err := coordinator.Acquire(context.Background(), direct, request) + if err != nil { + t.Fatalf("acquire direct identity: %v", err) + } + aliasRequest := mustRequest(t, throughAlias, OperationStore) + if _, err := coordinator.Acquire(context.Background(), throughAlias, aliasRequest); !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("expected alias acquisition to be nested, got %v", err) + } + if err := lease.Release(); err != nil { + t.Fatalf("release direct identity: %v", err) + } +} + +func mustIdentity(t *testing.T, path string) Identity { + t.Helper() + identity, err := ResolveIdentity(path) + if err != nil { + t.Fatalf("ResolveIdentity(%q): %v", path, err) + } + return identity +} + +func mustRequest(t *testing.T, identity Identity, operation Operation) Request { + t.Helper() + owner, err := NewOwner(operation, identity, "1.13.11", testOwnerStart) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + return Request{ + Operation: operation, + Mode: ModeExclusive, + Owner: owner, + } +} diff --git a/internal/coordination/control.go b/internal/coordination/control.go new file mode 100644 index 00000000..c2ac0e21 --- /dev/null +++ b/internal/coordination/control.go @@ -0,0 +1,87 @@ +package coordination + +import ( + "fmt" + "os" + "path/filepath" +) + +// PreparedControlNamespace identifies the canonical repository coordination +// directory and the deterministic paths reserved for future lock artifacts. +// Preparation does not create either artifact. +type PreparedControlNamespace struct { + Identity Identity + ControlDirectory string + LockArtifactPath string + OwnerMetadataPath string +} + +// PrepareControlNamespace creates and validates the repository coordination +// directory, then verifies that filesystem creation did not change the +// canonical repository identity. +func PrepareControlNamespace(containerDir string) (PreparedControlNamespace, error) { + return prepareControlNamespace(containerDir, ResolveIdentity) +} + +func prepareControlNamespace( + containerDir string, + resolve func(string) (Identity, error), +) (PreparedControlNamespace, error) { + identity, err := resolve(containerDir) + if err != nil { + return PreparedControlNamespace{}, err + } + if err := ValidateIdentity(identity); err != nil { + return PreparedControlNamespace{}, err + } + controlDirectory, err := ensureControlDirectory(identity) + if err != nil { + return PreparedControlNamespace{}, err + } + + finalIdentity, err := resolve(containerDir) + if err != nil { + return PreparedControlNamespace{}, err + } + if err := validateStableIdentity(identity, finalIdentity); err != nil { + return PreparedControlNamespace{}, err + } + + return PreparedControlNamespace{ + Identity: finalIdentity, + ControlDirectory: controlDirectory, + LockArtifactPath: filepath.Join(controlDirectory, LockArtifactName), + OwnerMetadataPath: filepath.Join(controlDirectory, OwnerMetadataName), + }, nil +} + +func ensureControlDirectory(identity Identity) (string, error) { + if err := os.MkdirAll(identity.CanonicalPath, 0o755); err != nil { + return "", fmt.Errorf("%w: create canonical container directory: %w", ErrRepositoryIdentityInvalid, err) + } + controlDirectory, err := ControlDirectory(identity) + if err != nil { + return "", err + } + if err := os.Mkdir(controlDirectory, 0o700); err != nil && !os.IsExist(err) { + return "", fmt.Errorf("%w: create repository control directory: %w", ErrRepositoryIdentityInvalid, err) + } + info, err := os.Lstat(controlDirectory) + if err != nil { + return "", fmt.Errorf("%w: inspect repository control directory: %w", ErrRepositoryIdentityInvalid, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return "", fmt.Errorf("%w: repository control path must be a real directory", ErrRepositoryIdentityInvalid) + } + return controlDirectory, nil +} + +func validateStableIdentity(initial, final Identity) error { + if err := ValidateIdentity(final); err != nil { + return err + } + if final != initial { + return fmt.Errorf("%w: canonical container namespace changed during preparation", ErrRepositoryIdentityInvalid) + } + return nil +} diff --git a/internal/coordination/control_test.go b/internal/coordination/control_test.go new file mode 100644 index 00000000..f9d99625 --- /dev/null +++ b/internal/coordination/control_test.go @@ -0,0 +1,215 @@ +package coordination + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestPrepareControlNamespaceCreatesMissingNamespace(t *testing.T) { + containerDir := filepath.Join(t.TempDir(), "repository", "containers") + + prepared, err := PrepareControlNamespace(containerDir) + if err != nil { + t.Fatalf("PrepareControlNamespace: %v", err) + } + if prepared.Identity.CanonicalPath != containerDir { + t.Fatalf("canonical path=%q want=%q", prepared.Identity.CanonicalPath, containerDir) + } + wantControlDirectory := filepath.Join(containerDir, ControlDirectoryName) + if prepared.ControlDirectory != wantControlDirectory { + t.Fatalf("control directory=%q want=%q", prepared.ControlDirectory, wantControlDirectory) + } + if prepared.LockArtifactPath != filepath.Join(wantControlDirectory, LockArtifactName) { + t.Fatalf("lock artifact path=%q", prepared.LockArtifactPath) + } + if prepared.OwnerMetadataPath != filepath.Join(wantControlDirectory, OwnerMetadataName) { + t.Fatalf("owner metadata path=%q", prepared.OwnerMetadataPath) + } + + info, err := os.Lstat(wantControlDirectory) + if err != nil { + t.Fatalf("inspect control directory: %v", err) + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + t.Fatalf("control path is not a real directory: mode=%v", info.Mode()) + } + if runtime.GOOS != "windows" && info.Mode().Perm() != 0o700 { + t.Fatalf("new control directory mode=%#o want=0700", info.Mode().Perm()) + } + assertNoCoordinationArtifacts(t, prepared) +} + +func TestPrepareControlNamespaceUsesDistinctCreationModes(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix-style permission bits are not authoritative on Windows") + } + + root := t.TempDir() + containerModeReference := filepath.Join(root, "container-mode-reference") + if err := os.Mkdir(containerModeReference, 0o755); err != nil { + t.Fatalf("create container mode reference: %v", err) + } + controlModeReference := filepath.Join(root, "control-mode-reference") + if err := os.Mkdir(controlModeReference, 0o700); err != nil { + t.Fatalf("create control mode reference: %v", err) + } + + containerDir := filepath.Join(root, "missing-parent", "containers") + prepared, err := PrepareControlNamespace(containerDir) + if err != nil { + t.Fatalf("PrepareControlNamespace: %v", err) + } + wantContainerMode := mustDirectoryMode(t, containerModeReference) + for _, path := range []string{filepath.Dir(containerDir), containerDir} { + if got := mustDirectoryMode(t, path); got != wantContainerMode { + t.Fatalf("directory %q mode=%#o want container mode %#o", filepath.Base(path), got, wantContainerMode) + } + } + if got, want := mustDirectoryMode(t, prepared.ControlDirectory), mustDirectoryMode(t, controlModeReference); got != want { + t.Fatalf("control directory mode=%#o want control mode %#o", got, want) + } +} + +func TestPrepareControlNamespaceAcceptsExistingDirectoryAndIsIdempotent(t *testing.T) { + containerDir := t.TempDir() + controlDirectory := filepath.Join(containerDir, ControlDirectoryName) + if err := os.Mkdir(controlDirectory, 0o750); err != nil { + t.Fatalf("create control directory: %v", err) + } + before, err := os.Stat(controlDirectory) + if err != nil { + t.Fatalf("stat control directory before preparation: %v", err) + } + + first, err := PrepareControlNamespace(containerDir) + if err != nil { + t.Fatalf("first PrepareControlNamespace: %v", err) + } + second, err := PrepareControlNamespace(containerDir) + if err != nil { + t.Fatalf("second PrepareControlNamespace: %v", err) + } + if first != second { + t.Fatalf("idempotent preparation changed result: first=%+v second=%+v", first, second) + } + after, err := os.Stat(controlDirectory) + if err != nil { + t.Fatalf("stat control directory after preparation: %v", err) + } + if after.Mode().Perm() != before.Mode().Perm() { + t.Fatalf("existing control directory mode changed from %#o to %#o", before.Mode().Perm(), after.Mode().Perm()) + } + assertNoCoordinationArtifacts(t, second) +} + +func TestPrepareControlNamespaceRejectsRegularFile(t *testing.T) { + containerDir := t.TempDir() + controlPath := filepath.Join(containerDir, ControlDirectoryName) + if err := os.WriteFile(controlPath, []byte("unsafe"), 0o600); err != nil { + t.Fatalf("create control file: %v", err) + } + + _, err := PrepareControlNamespace(containerDir) + if !errors.Is(err, ErrRepositoryIdentityInvalid) { + t.Fatalf("error=%v, want invalid repository identity", err) + } +} + +func TestPrepareControlNamespaceRejectsSymlink(t *testing.T) { + root := t.TempDir() + containerDir := filepath.Join(root, "containers") + target := filepath.Join(root, "outside") + if err := os.MkdirAll(containerDir, 0o755); err != nil { + t.Fatalf("create container directory: %v", err) + } + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatalf("create symlink target: %v", err) + } + if err := os.Symlink(target, filepath.Join(containerDir, ControlDirectoryName)); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + _, err := PrepareControlNamespace(containerDir) + if !errors.Is(err, ErrRepositoryIdentityInvalid) { + t.Fatalf("error=%v, want invalid repository identity", err) + } + for _, artifact := range []string{LockArtifactName, OwnerMetadataName} { + if _, err := os.Lstat(filepath.Join(target, artifact)); !os.IsNotExist(err) { + t.Fatalf("unsafe target artifact %q exists, stat err=%v", artifact, err) + } + } +} + +func TestPrepareControlNamespacePreservesExistingSymlinkPrefixIdentity(t *testing.T) { + root := t.TempDir() + realParent := filepath.Join(root, "real") + if err := os.MkdirAll(realParent, 0o755); err != nil { + t.Fatalf("create real parent: %v", err) + } + alias := filepath.Join(root, "alias") + if err := os.Symlink(realParent, alias); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + configured := filepath.Join(alias, "repository", "containers") + + prepared, err := PrepareControlNamespace(configured) + if err != nil { + t.Fatalf("PrepareControlNamespace: %v", err) + } + wantCanonical := filepath.Join(realParent, "repository", "containers") + if prepared.Identity.CanonicalPath != wantCanonical { + t.Fatalf("canonical path=%q want=%q", prepared.Identity.CanonicalPath, wantCanonical) + } + if prepared.ControlDirectory != filepath.Join(wantCanonical, ControlDirectoryName) { + t.Fatalf("control directory=%q", prepared.ControlDirectory) + } + assertNoCoordinationArtifacts(t, prepared) +} + +func TestPrepareControlNamespaceRejectsIdentityChangeAfterCreation(t *testing.T) { + initial := mustIdentity(t, filepath.Join(t.TempDir(), "containers")) + final := mustIdentity(t, t.TempDir()) + calls := 0 + resolver := func(string) (Identity, error) { + calls++ + if calls == 1 { + return initial, nil + } + return final, nil + } + + _, err := prepareControlNamespace("configured-container-directory", resolver) + if !errors.Is(err, ErrRepositoryIdentityInvalid) { + t.Fatalf("error=%v, want invalid repository identity", err) + } + if calls != 2 { + t.Fatalf("identity resolver calls=%d want=2", calls) + } + if _, err := os.Stat(filepath.Join(initial.CanonicalPath, ControlDirectoryName)); err != nil { + t.Fatalf("control directory was not prepared before final resolution: %v", err) + } +} + +func assertNoCoordinationArtifacts(t *testing.T, prepared PreparedControlNamespace) { + t.Helper() + for _, path := range []string{prepared.LockArtifactPath, prepared.OwnerMetadataPath} { + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("preparation created artifact %q, stat err=%v", filepath.Base(path), err) + } + } +} + +func mustDirectoryMode(t *testing.T, path string) os.FileMode { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat directory %q: %v", filepath.Base(path), err) + } + if !info.IsDir() { + t.Fatalf("path %q is not a directory", filepath.Base(path)) + } + return info.Mode().Perm() +} diff --git a/internal/coordination/coordinator.go b/internal/coordination/coordinator.go new file mode 100644 index 00000000..0ae3c289 --- /dev/null +++ b/internal/coordination/coordinator.go @@ -0,0 +1,151 @@ +package coordination + +import ( + "context" + "errors" + "fmt" + "sync" +) + +var productionProcessRegistry processRegistry + +// NewCoordinator returns the production repository coordinator. All instances +// share one process registry so same-process acquisition is non-reentrant even +// when callers construct separate coordinators. +func NewCoordinator() Coordinator { + return newRepositoryCoordinator(coordinatorDependencies{ + prepare: PrepareControlNamespace, + reserve: reserveProductionProcessIdentity, + acquireNative: acquireNativeLockResource, + publishOwner: publishOwnerMetadata, + removeOwner: removeOwnerMetadata, + }) +} + +func reserveProductionProcessIdentity(identity Identity) (processReservationResource, error) { + return productionProcessRegistry.reserve(identity) +} + +type processReservationResource interface { + release() +} + +type nativeLockResource interface { + release() error +} + +type coordinatorDependencies struct { + prepare func(string) (PreparedControlNamespace, error) + reserve func(Identity) (processReservationResource, error) + acquireNative func(PreparedControlNamespace) (nativeLockResource, error) + publishOwner func(PreparedControlNamespace, Owner) error + removeOwner func(PreparedControlNamespace) error +} + +type repositoryCoordinator struct { + dependencies coordinatorDependencies +} + +func newRepositoryCoordinator(dependencies coordinatorDependencies) *repositoryCoordinator { + return &repositoryCoordinator{dependencies: dependencies} +} + +func acquireNativeLockResource(prepared PreparedControlNamespace) (nativeLockResource, error) { + return acquireNativeLock(prepared) +} + +func (coordinator *repositoryCoordinator) Acquire( + ctx context.Context, + identity Identity, + request Request, +) (Lease, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if err := ValidateRequest(identity, request); err != nil { + return nil, err + } + + prepared, err := coordinator.prepareAcquisition(identity) + if err != nil { + return nil, err + } + + reservation, err := coordinator.dependencies.reserve(prepared.Identity) + if err != nil { + return nil, err + } + + nativeLock, err := coordinator.dependencies.acquireNative(prepared) + if err != nil { + reservation.release() + return nil, err + } + + if err := coordinator.dependencies.publishOwner(prepared, request.Owner); err != nil { + return nil, releaseFailedOwnerPublication(err, nativeLock, reservation) + } + + return &repositoryLease{ + prepared: prepared, + reservation: reservation, + nativeLock: nativeLock, + removeOwner: coordinator.dependencies.removeOwner, + }, nil +} + +func (coordinator *repositoryCoordinator) prepareAcquisition(identity Identity) (PreparedControlNamespace, error) { + prepared, err := coordinator.dependencies.prepare(identity.CanonicalPath) + if err != nil { + return PreparedControlNamespace{}, err + } + if prepared.Identity != identity { + return PreparedControlNamespace{}, fmt.Errorf("%w: prepared repository identity does not match acquisition identity", ErrRepositoryIdentityInvalid) + } + return prepared, nil +} + +func releaseFailedOwnerPublication(ownerErr error, nativeLock nativeLockResource, reservation processReservationResource) error { + nativeReleaseErr := nativeLock.release() + reservation.release() + if nativeReleaseErr != nil { + return errors.Join(ownerErr, nativeReleaseErr) + } + return ownerErr +} + +type repositoryLease struct { + prepared PreparedControlNamespace + reservation processReservationResource + nativeLock nativeLockResource + removeOwner func(PreparedControlNamespace) error + + releaseOnce sync.Once + releaseErr error +} + +func (lease *repositoryLease) Release() error { + if lease == nil { + return nil + } + lease.releaseOnce.Do(func() { + ownerRemovalErr := lease.removeOwner(lease.prepared) + nativeReleaseErr := lease.nativeLock.release() + lease.reservation.release() + + switch { + case nativeReleaseErr != nil && ownerRemovalErr != nil: + lease.releaseErr = errors.Join(nativeReleaseErr, ownerRemovalErr) + case nativeReleaseErr != nil: + lease.releaseErr = nativeReleaseErr + default: + // Owner metadata is diagnostic. Its removal failure alone must not + // turn a successfully released native lease into an operation error. + lease.releaseErr = nil + } + }) + return lease.releaseErr +} diff --git a/internal/coordination/coordinator_native_test.go b/internal/coordination/coordinator_native_test.go new file mode 100644 index 00000000..488182de --- /dev/null +++ b/internal/coordination/coordinator_native_test.go @@ -0,0 +1,163 @@ +//go:build linux || darwin || windows + +package coordination + +import ( + "context" + "errors" + "os" + "testing" + "time" +) + +func TestProductionCoordinatorIntegratedNativeLifecycle(t *testing.T) { + repositoryPath := t.TempDir() + identity, request := mustProductionRequest(t, repositoryPath, OperationStore, time.Unix(1_700_000_000, 0)) + coordinator := NewCoordinator() + + lease, err := coordinator.Acquire(context.Background(), identity, request) + if err != nil { + t.Fatalf("Acquire: %v", err) + } + t.Cleanup(func() { _ = lease.Release() }) + + prepared, err := PrepareControlNamespace(repositoryPath) + if err != nil { + t.Fatalf("PrepareControlNamespace: %v", err) + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("lstat repository.lock: %v", err) + } else if !info.Mode().IsRegular() { + t.Fatalf("repository.lock mode=%v want regular", info.Mode()) + } + publishedOwner, err := readOwnerMetadata(prepared) + if err != nil { + t.Fatalf("readOwnerMetadata: %v", err) + } + if publishedOwner != request.Owner { + t.Fatalf("published owner=%+v want=%+v", publishedOwner, request.Owner) + } + + if err := lease.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("persistent repository.lock missing: %v", err) + } else if !info.Mode().IsRegular() { + t.Fatalf("repository.lock mode after release=%v want regular", info.Mode()) + } + if _, err := os.Lstat(prepared.OwnerMetadataPath); !os.IsNotExist(err) { + t.Fatalf("owner metadata exists after release, stat err=%v", err) + } + + reacquired, err := NewCoordinator().Acquire(context.Background(), identity, request) + if err != nil { + t.Fatalf("reacquire: %v", err) + } + if err := reacquired.Release(); err != nil { + t.Fatalf("release reacquired Lease: %v", err) + } +} + +func TestProductionCoordinatorsShareProcessRegistryAndProtectSuccessor(t *testing.T) { + repositoryPath := t.TempDir() + identity, firstRequest := mustProductionRequest(t, repositoryPath, OperationStore, time.Unix(1_700_000_000, 0)) + _, successorRequest := mustProductionRequest(t, repositoryPath, OperationRestore, time.Unix(1_700_000_001, 0)) + firstCoordinator := NewCoordinator() + secondCoordinator := NewCoordinator() + + firstLease, err := firstCoordinator.Acquire(context.Background(), identity, firstRequest) + if err != nil { + t.Fatalf("first Acquire: %v", err) + } + t.Cleanup(func() { _ = firstLease.Release() }) + + nestedLease, err := secondCoordinator.Acquire(context.Background(), identity, successorRequest) + if nestedLease != nil { + _ = nestedLease.Release() + t.Fatal("nested production acquisition returned a Lease") + } + if !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("nested Acquire error=%v want ErrNestedRepositoryAcquisition", err) + } + if errors.Is(err, ErrRepositoryBusy) { + t.Fatalf("nested Acquire error=%v unexpectedly classified Busy", err) + } + + if err := firstLease.Release(); err != nil { + t.Fatalf("release first Lease: %v", err) + } + successorLease, err := secondCoordinator.Acquire(context.Background(), identity, successorRequest) + if err != nil { + t.Fatalf("successor Acquire: %v", err) + } + t.Cleanup(func() { _ = successorLease.Release() }) + + if err := firstLease.Release(); err != nil { + t.Fatalf("stale first Release: %v", err) + } + prepared, err := PrepareControlNamespace(repositoryPath) + if err != nil { + t.Fatalf("PrepareControlNamespace: %v", err) + } + publishedOwner, err := readOwnerMetadata(prepared) + if err != nil { + t.Fatalf("read successor owner metadata: %v", err) + } + if publishedOwner != successorRequest.Owner { + t.Fatalf("owner after stale release=%+v want successor=%+v", publishedOwner, successorRequest.Owner) + } + + thirdLease, err := NewCoordinator().Acquire(context.Background(), identity, firstRequest) + if thirdLease != nil { + _ = thirdLease.Release() + t.Fatal("stale release freed successor process reservation") + } + if !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("Acquire after stale release error=%v want ErrNestedRepositoryAcquisition", err) + } + if err := successorLease.Release(); err != nil { + t.Fatalf("release successor Lease: %v", err) + } +} + +func TestProductionCoordinatorAllowsDifferentRepositories(t *testing.T) { + firstIdentity, firstRequest := mustProductionRequest(t, t.TempDir(), OperationStore, time.Unix(1_700_000_000, 0)) + secondIdentity, secondRequest := mustProductionRequest(t, t.TempDir(), OperationStore, time.Unix(1_700_000_000, 0)) + + firstLease, err := NewCoordinator().Acquire(context.Background(), firstIdentity, firstRequest) + if err != nil { + t.Fatalf("acquire first repository: %v", err) + } + t.Cleanup(func() { _ = firstLease.Release() }) + secondLease, err := NewCoordinator().Acquire(context.Background(), secondIdentity, secondRequest) + if err != nil { + t.Fatalf("acquire second repository: %v", err) + } + t.Cleanup(func() { _ = secondLease.Release() }) + + if err := secondLease.Release(); err != nil { + t.Fatalf("release second repository: %v", err) + } + if err := firstLease.Release(); err != nil { + t.Fatalf("release first repository: %v", err) + } +} + +func mustProductionRequest( + t *testing.T, + repositoryPath string, + operation Operation, + startedAt time.Time, +) (Identity, Request) { + t.Helper() + identity, err := ResolveIdentity(repositoryPath) + if err != nil { + t.Fatalf("ResolveIdentity: %v", err) + } + owner, err := NewOwner(operation, identity, "test-version", startedAt) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + return identity, Request{Operation: operation, Mode: ModeExclusive, Owner: owner} +} diff --git a/internal/coordination/coordinator_test.go b/internal/coordination/coordinator_test.go new file mode 100644 index 00000000..0ec243f4 --- /dev/null +++ b/internal/coordination/coordinator_test.go @@ -0,0 +1,389 @@ +package coordination + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" + "time" +) + +func TestRepositoryCoordinatorAcquisitionAndReleaseOrdering(t *testing.T) { + fixture := newCoordinatorFixture(t) + trace := &coordinatorTrace{} + coordinator := newRepositoryCoordinator(coordinatorDependencies{ + prepare: func(path string) (PreparedControlNamespace, error) { + trace.add("prepare") + if path != fixture.identity.CanonicalPath { + t.Fatalf("prepare path=%q want=%q", path, fixture.identity.CanonicalPath) + } + return fixture.prepared, nil + }, + reserve: func(identity Identity) (processReservationResource, error) { + trace.add("reserve") + return &fakeProcessReservation{releaseFn: func() { trace.add("reservation release") }}, nil + }, + acquireNative: func(prepared PreparedControlNamespace) (nativeLockResource, error) { + trace.add("native acquire") + return &fakeNativeLock{releaseFn: func() error { + trace.add("native release") + return nil + }}, nil + }, + publishOwner: func(prepared PreparedControlNamespace, owner Owner) error { + trace.add("owner publish") + return nil + }, + removeOwner: func(prepared PreparedControlNamespace) error { + trace.add("owner remove") + return nil + }, + }) + + lease, err := coordinator.Acquire(context.Background(), fixture.identity, fixture.request) + if err != nil { + t.Fatalf("Acquire: %v", err) + } + if lease == nil { + t.Fatal("Acquire returned nil Lease") + } + trace.require(t, []string{"prepare", "reserve", "native acquire", "owner publish"}) + + if err := lease.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + trace.require(t, []string{ + "prepare", "reserve", "native acquire", "owner publish", + "owner remove", "native release", "reservation release", + }) +} + +func TestRepositoryCoordinatorAcquisitionFailureUnwind(t *testing.T) { + prepareErr := errors.New("prepare failure") + reserveErr := errors.New("reserve failure") + nativeErr := ErrRepositoryBusy + publishErr := errors.New("owner publication failure") + nativeReleaseErr := errors.New("native release failure") + + tests := []struct { + name string + configure func(*coordinatorDependencies, *coordinatorTrace) + want []string + wantErrs []error + }{ + { + name: "prepare failure stops acquisition", + configure: func(dependencies *coordinatorDependencies, trace *coordinatorTrace) { + dependencies.prepare = func(string) (PreparedControlNamespace, error) { + trace.add("prepare") + return PreparedControlNamespace{}, prepareErr + } + }, + want: []string{"prepare"}, + wantErrs: []error{prepareErr}, + }, + { + name: "reservation failure stops native acquisition", + configure: func(dependencies *coordinatorDependencies, trace *coordinatorTrace) { + dependencies.reserve = func(Identity) (processReservationResource, error) { + trace.add("reserve") + return nil, reserveErr + } + }, + want: []string{"prepare", "reserve"}, + wantErrs: []error{reserveErr}, + }, + { + name: "native failure releases reservation", + configure: func(dependencies *coordinatorDependencies, trace *coordinatorTrace) { + dependencies.acquireNative = func(PreparedControlNamespace) (nativeLockResource, error) { + trace.add("native acquire") + return nil, nativeErr + } + }, + want: []string{"prepare", "reserve", "native acquire", "reservation release"}, + wantErrs: []error{nativeErr}, + }, + { + name: "owner failure releases native then reservation", + configure: func(dependencies *coordinatorDependencies, trace *coordinatorTrace) { + dependencies.publishOwner = func(PreparedControlNamespace, Owner) error { + trace.add("owner publish") + return publishErr + } + }, + want: []string{ + "prepare", "reserve", "native acquire", "owner publish", + "native release", "reservation release", + }, + wantErrs: []error{publishErr}, + }, + { + name: "owner and native cleanup failures are both preserved", + configure: func(dependencies *coordinatorDependencies, trace *coordinatorTrace) { + dependencies.publishOwner = func(PreparedControlNamespace, Owner) error { + trace.add("owner publish") + return publishErr + } + dependencies.acquireNative = func(PreparedControlNamespace) (nativeLockResource, error) { + trace.add("native acquire") + return &fakeNativeLock{releaseFn: func() error { + trace.add("native release") + return nativeReleaseErr + }}, nil + } + }, + want: []string{ + "prepare", "reserve", "native acquire", "owner publish", + "native release", "reservation release", + }, + wantErrs: []error{publishErr, nativeReleaseErr}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := newCoordinatorFixture(t) + trace := &coordinatorTrace{} + dependencies := successfulCoordinatorDependencies(fixture.prepared, trace) + test.configure(&dependencies, trace) + coordinator := newRepositoryCoordinator(dependencies) + + lease, err := coordinator.Acquire(context.Background(), fixture.identity, fixture.request) + if lease != nil { + _ = lease.Release() + t.Fatal("failed acquisition returned a Lease") + } + if err == nil { + t.Fatal("failed acquisition returned nil error") + } + for _, wantErr := range test.wantErrs { + if !errors.Is(err, wantErr) { + t.Fatalf("Acquire error=%v want errors.Is(%v)", err, wantErr) + } + } + trace.require(t, test.want) + }) + } +} + +func TestRepositoryCoordinatorHonorsCanceledContextBeforeSideEffects(t *testing.T) { + fixture := newCoordinatorFixture(t) + trace := &coordinatorTrace{} + coordinator := newRepositoryCoordinator(successfulCoordinatorDependencies(fixture.prepared, trace)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + lease, err := coordinator.Acquire(ctx, fixture.identity, fixture.request) + if lease != nil { + t.Fatal("canceled acquisition returned a Lease") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("Acquire error=%v want context.Canceled", err) + } + trace.require(t, nil) +} + +func TestRepositoryLeaseDiagnosticRemovalAndNativeFailureSemantics(t *testing.T) { + ownerRemovalErr := errors.New("owner removal failure") + nativeReleaseErr := errors.New("native release failure") + + tests := []struct { + name string + ownerRemovalErr error + nativeReleaseErr error + wantErrs []error + }{ + {name: "owner removal alone is non-fatal", ownerRemovalErr: ownerRemovalErr}, + {name: "native release is authoritative", nativeReleaseErr: nativeReleaseErr, wantErrs: []error{nativeReleaseErr}}, + { + name: "owner and native failures are joined", + ownerRemovalErr: ownerRemovalErr, + nativeReleaseErr: nativeReleaseErr, + wantErrs: []error{ownerRemovalErr, nativeReleaseErr}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := newCoordinatorFixture(t) + trace := &coordinatorTrace{} + dependencies := successfulCoordinatorDependencies(fixture.prepared, trace) + dependencies.removeOwner = func(PreparedControlNamespace) error { + trace.add("owner remove") + return test.ownerRemovalErr + } + dependencies.acquireNative = func(PreparedControlNamespace) (nativeLockResource, error) { + trace.add("native acquire") + return &fakeNativeLock{releaseFn: func() error { + trace.add("native release") + return test.nativeReleaseErr + }}, nil + } + coordinator := newRepositoryCoordinator(dependencies) + lease, err := coordinator.Acquire(context.Background(), fixture.identity, fixture.request) + if err != nil { + t.Fatalf("Acquire: %v", err) + } + + err = lease.Release() + if len(test.wantErrs) == 0 && err != nil { + t.Fatalf("Release error=%v want nil", err) + } + for _, wantErr := range test.wantErrs { + if !errors.Is(err, wantErr) { + t.Fatalf("Release error=%v want errors.Is(%v)", err, wantErr) + } + } + trace.require(t, []string{ + "prepare", "reserve", "native acquire", "owner publish", + "owner remove", "native release", "reservation release", + }) + }) + } +} + +func TestRepositoryLeaseConcurrentReleaseRunsLifecycleOnce(t *testing.T) { + fixture := newCoordinatorFixture(t) + trace := &coordinatorTrace{} + coordinator := newRepositoryCoordinator(successfulCoordinatorDependencies(fixture.prepared, trace)) + lease, err := coordinator.Acquire(context.Background(), fixture.identity, fixture.request) + if err != nil { + t.Fatalf("Acquire: %v", err) + } + + const releasers = 32 + errorsByRelease := make(chan error, releasers) + var workers sync.WaitGroup + workers.Add(releasers) + for range releasers { + go func() { + defer workers.Done() + errorsByRelease <- lease.Release() + }() + } + workers.Wait() + close(errorsByRelease) + for err := range errorsByRelease { + if err != nil { + t.Fatalf("concurrent Release: %v", err) + } + } + + trace.require(t, []string{ + "prepare", "reserve", "native acquire", "owner publish", + "owner remove", "native release", "reservation release", + }) +} + +type coordinatorFixture struct { + identity Identity + prepared PreparedControlNamespace + request Request +} + +func newCoordinatorFixture(t *testing.T) coordinatorFixture { + t.Helper() + identity, err := ResolveIdentity(t.TempDir()) + if err != nil { + t.Fatalf("ResolveIdentity: %v", err) + } + controlDirectory, err := ControlDirectory(identity) + if err != nil { + t.Fatalf("ControlDirectory: %v", err) + } + owner, err := NewOwner(OperationStore, identity, "test-version", time.Unix(1_700_000_000, 0)) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + return coordinatorFixture{ + identity: identity, + prepared: PreparedControlNamespace{ + Identity: identity, + ControlDirectory: controlDirectory, + LockArtifactPath: filepath.Join(controlDirectory, LockArtifactName), + OwnerMetadataPath: filepath.Join(controlDirectory, OwnerMetadataName), + }, + request: Request{Operation: OperationStore, Mode: ModeExclusive, Owner: owner}, + } +} + +func successfulCoordinatorDependencies( + prepared PreparedControlNamespace, + trace *coordinatorTrace, +) coordinatorDependencies { + return coordinatorDependencies{ + prepare: func(string) (PreparedControlNamespace, error) { + trace.add("prepare") + return prepared, nil + }, + reserve: func(Identity) (processReservationResource, error) { + trace.add("reserve") + return &fakeProcessReservation{releaseFn: func() { trace.add("reservation release") }}, nil + }, + acquireNative: func(PreparedControlNamespace) (nativeLockResource, error) { + trace.add("native acquire") + return &fakeNativeLock{releaseFn: func() error { + trace.add("native release") + return nil + }}, nil + }, + publishOwner: func(PreparedControlNamespace, Owner) error { + trace.add("owner publish") + return nil + }, + removeOwner: func(PreparedControlNamespace) error { + trace.add("owner remove") + return nil + }, + } +} + +type fakeProcessReservation struct { + releaseOnce sync.Once + releaseFn func() +} + +func (reservation *fakeProcessReservation) release() { + reservation.releaseOnce.Do(reservation.releaseFn) +} + +type fakeNativeLock struct { + releaseOnce sync.Once + releaseFn func() error + releaseErr error +} + +func (lock *fakeNativeLock) release() error { + lock.releaseOnce.Do(func() { + lock.releaseErr = lock.releaseFn() + }) + return lock.releaseErr +} + +type coordinatorTrace struct { + mu sync.Mutex + events []string +} + +func (trace *coordinatorTrace) add(event string) { + trace.mu.Lock() + defer trace.mu.Unlock() + trace.events = append(trace.events, event) +} + +func (trace *coordinatorTrace) require(t *testing.T, want []string) { + t.Helper() + trace.mu.Lock() + got := append([]string(nil), trace.events...) + trace.mu.Unlock() + if len(got) != len(want) { + t.Fatalf("trace=%v want=%v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("trace=%v want=%v", got, want) + } + } +} diff --git a/internal/coordination/errors.go b/internal/coordination/errors.go new file mode 100644 index 00000000..7dbf30fd --- /dev/null +++ b/internal/coordination/errors.go @@ -0,0 +1,16 @@ +package coordination + +import "errors" + +var ( + // ErrRepositoryBusy reports that another holder owns the repository lease. + ErrRepositoryBusy = errors.New("repository is busy") + // ErrRepositoryLockUnsupported reports that repository coordination is not + // supported by the active platform or filesystem. + ErrRepositoryLockUnsupported = errors.New("repository coordination is unsupported") + // ErrRepositoryIdentityInvalid reports an invalid repository namespace. + ErrRepositoryIdentityInvalid = errors.New("repository identity is invalid") + // ErrNestedRepositoryAcquisition reports a repeated acquisition of an + // already-held repository identity. + ErrNestedRepositoryAcquisition = errors.New("nested repository acquisition is not allowed") +) diff --git a/internal/coordination/identity.go b/internal/coordination/identity.go new file mode 100644 index 00000000..25570358 --- /dev/null +++ b/internal/coordination/identity.go @@ -0,0 +1,134 @@ +package coordination + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +const ( + // ControlDirectoryName is deliberately a directory because startup recovery + // treats non-directory entries in the container root as orphan containers. + ControlDirectoryName = ".coldkeep-control" + LockArtifactName = "repository.lock" + OwnerMetadataName = "owner.json" +) + +// Identity is the canonical same-host container namespace used by repository +// coordination. CanonicalPath remains internal; diagnostics must expose Hash. +type Identity struct { + CanonicalPath string + Hash string +} + +// ResolveIdentity derives a non-mutating identity from a container directory. +// +// Existing path components have symlinks resolved. Missing leaf components are +// appended to the resolved nearest existing ancestor; Phase 12 owns creation +// and final re-resolution of the control directory. +func ResolveIdentity(containerDir string) (Identity, error) { + if strings.TrimSpace(containerDir) == "" || strings.ContainsRune(containerDir, '\x00') { + return Identity{}, fmt.Errorf("%w: container directory is empty or contains NUL", ErrRepositoryIdentityInvalid) + } + if isUNCPath(containerDir) { + return Identity{}, fmt.Errorf("%w: UNC and network paths are outside the local-filesystem contract", ErrRepositoryLockUnsupported) + } + + absolute, err := filepath.Abs(filepath.Clean(containerDir)) + if err != nil { + return Identity{}, fmt.Errorf("%w: make container directory absolute: %w", ErrRepositoryIdentityInvalid, err) + } + canonical, err := resolveExistingPathPrefix(absolute) + if err != nil { + return Identity{}, err + } + canonical = normalizePlatformPath(filepath.Clean(canonical)) + + sum := sha256.Sum256([]byte(canonical)) + return Identity{ + CanonicalPath: canonical, + Hash: hex.EncodeToString(sum[:]), + }, nil +} + +// ValidateIdentity checks that an identity is internally consistent. +func ValidateIdentity(identity Identity) error { + if strings.TrimSpace(identity.CanonicalPath) == "" || strings.ContainsRune(identity.CanonicalPath, '\x00') { + return fmt.Errorf("%w: canonical path is empty or contains NUL", ErrRepositoryIdentityInvalid) + } + if !filepath.IsAbs(identity.CanonicalPath) { + return fmt.Errorf("%w: canonical path must be absolute", ErrRepositoryIdentityInvalid) + } + sum := sha256.Sum256([]byte(identity.CanonicalPath)) + if identity.Hash != hex.EncodeToString(sum[:]) { + return fmt.Errorf("%w: identity hash does not match canonical path", ErrRepositoryIdentityInvalid) + } + return nil +} + +// ControlDirectory returns the fixed lock namespace without creating it. +func ControlDirectory(identity Identity) (string, error) { + if err := ValidateIdentity(identity); err != nil { + return "", err + } + return filepath.Join(identity.CanonicalPath, ControlDirectoryName), nil +} + +func resolveExistingPathPrefix(absolute string) (string, error) { + resolved, err := filepath.EvalSymlinks(absolute) + if err == nil { + return resolved, nil + } + if !os.IsNotExist(err) { + return "", fmt.Errorf("%w: resolve container directory: %w", ErrRepositoryIdentityInvalid, err) + } + + current := absolute + missing := make([]string, 0, 4) + for { + if _, statErr := os.Lstat(current); statErr == nil { + resolvedPrefix, evalErr := filepath.EvalSymlinks(current) + if evalErr != nil { + return "", fmt.Errorf("%w: resolve existing container ancestor: %w", ErrRepositoryIdentityInvalid, evalErr) + } + parts := append([]string{resolvedPrefix}, reverseStrings(missing)...) + return filepath.Join(parts...), nil + } else if !os.IsNotExist(statErr) { + return "", fmt.Errorf("%w: inspect container ancestor: %w", ErrRepositoryIdentityInvalid, statErr) + } + + parent := filepath.Dir(current) + if parent == current { + return "", fmt.Errorf("%w: no existing container ancestor", ErrRepositoryIdentityInvalid) + } + missing = append(missing, filepath.Base(current)) + current = parent + } +} + +func reverseStrings(values []string) []string { + reversed := make([]string, len(values)) + for i := range values { + reversed[len(values)-1-i] = values[i] + } + return reversed +} + +func normalizePlatformPath(path string) string { + if runtime.GOOS != "windows" { + return path + } + volume := filepath.VolumeName(path) + if len(volume) == 2 && volume[1] == ':' { + return strings.ToUpper(volume[:1]) + path[1:] + } + return path +} + +func isUNCPath(path string) bool { + return strings.HasPrefix(path, `\\`) || strings.HasPrefix(path, "//") +} diff --git a/internal/coordination/identity_test.go b/internal/coordination/identity_test.go new file mode 100644 index 00000000..6550a829 --- /dev/null +++ b/internal/coordination/identity_test.go @@ -0,0 +1,157 @@ +package coordination + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveIdentityNormalizesAliasesAndTrailingSeparators(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "repository", "containers") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatalf("create target: %v", err) + } + alias := filepath.Join(root, "alias") + if err := os.Symlink(filepath.Join(root, "repository"), alias); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + direct := mustIdentity(t, target) + throughAlias := mustIdentity(t, filepath.Join(alias, "containers")+string(filepath.Separator)) + if direct != throughAlias { + t.Fatalf("alias identity mismatch direct=%+v alias=%+v", direct, throughAlias) + } + if !filepath.IsAbs(direct.CanonicalPath) { + t.Fatalf("canonical path is not absolute: %q", direct.CanonicalPath) + } +} + +func TestResolveIdentityNormalizesRelativeAndAbsoluteForms(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "repository", "containers") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatalf("create target: %v", err) + } + previous, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + if err := os.Chdir(root); err != nil { + t.Fatalf("change working directory: %v", err) + } + t.Cleanup(func() { + if err := os.Chdir(previous); err != nil { + t.Errorf("restore working directory: %v", err) + } + }) + + relative := mustIdentity(t, filepath.Join("repository", "containers")) + absolute := mustIdentity(t, target) + if relative != absolute { + t.Fatalf("relative identity=%+v absolute identity=%+v", relative, absolute) + } +} + +func TestResolveIdentityUsesResolvedNearestExistingAncestor(t *testing.T) { + root := t.TempDir() + realParent := filepath.Join(root, "real") + if err := os.MkdirAll(realParent, 0o755); err != nil { + t.Fatalf("create real parent: %v", err) + } + alias := filepath.Join(root, "alias") + if err := os.Symlink(realParent, alias); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + identity := mustIdentity(t, filepath.Join(alias, "missing", "containers")) + want := filepath.Join(realParent, "missing", "containers") + if identity.CanonicalPath != want { + t.Fatalf("canonical path=%q want=%q", identity.CanonicalPath, want) + } + if _, err := os.Stat(filepath.Join(realParent, "missing")); !os.IsNotExist(err) { + t.Fatalf("identity resolution must not create missing directories, stat err=%v", err) + } +} + +func TestResolveIdentityRejectsInvalidAndNetworkPaths(t *testing.T) { + for _, path := range []string{"", " \t ", "bad\x00path"} { + _, err := ResolveIdentity(path) + if !errors.Is(err, ErrRepositoryIdentityInvalid) { + t.Fatalf("ResolveIdentity(%q) error=%v, want invalid identity", path, err) + } + } + + for _, path := range []string{"//server/share", `\\server\share`} { + _, err := ResolveIdentity(path) + if !errors.Is(err, ErrRepositoryLockUnsupported) { + t.Fatalf("ResolveIdentity(%q) error=%v, want unsupported", path, err) + } + } +} + +func TestIdentityResolutionPreservesUnderlyingFilesystemCause(t *testing.T) { + root := t.TempDir() + alias := filepath.Join(root, "broken-alias") + if err := os.Symlink(filepath.Join(root, "missing-target"), alias); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + _, err := ResolveIdentity(filepath.Join(alias, "containers")) + if !errors.Is(err, ErrRepositoryIdentityInvalid) { + t.Fatalf("expected invalid identity classification, got %v", err) + } + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("expected underlying filesystem cause, got %v", err) + } +} + +func TestIdentityHashIsStableAndPathFree(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + again := mustIdentity(t, identity.CanonicalPath) + if identity.Hash != again.Hash { + t.Fatalf("identity hash changed: %q != %q", identity.Hash, again.Hash) + } + if len(identity.Hash) != sha256HexLength || !isLowerHex(identity.Hash) { + t.Fatalf("identity hash is not lowercase SHA-256: %q", identity.Hash) + } + if strings.Contains(identity.Hash, identity.CanonicalPath) { + t.Fatal("identity hash exposed canonical path") + } +} + +func TestResolveIdentitySeparatesDistinctRepositories(t *testing.T) { + first := mustIdentity(t, t.TempDir()) + second := mustIdentity(t, t.TempDir()) + if first == second || first.Hash == second.Hash { + t.Fatalf("distinct repositories shared an identity: first=%+v second=%+v", first, second) + } +} + +func TestCoordinationControlDirectoryUsesRecoverySafeSubdirectory(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + controlDir, err := ControlDirectory(identity) + if err != nil { + t.Fatalf("ControlDirectory: %v", err) + } + if filepath.Dir(controlDir) != identity.CanonicalPath || filepath.Base(controlDir) != ControlDirectoryName { + t.Fatalf("unexpected control directory %q", controlDir) + } + if _, err := os.Stat(controlDir); !os.IsNotExist(err) { + t.Fatalf("contract helper must not create control directory, stat err=%v", err) + } + if strings.ContainsAny(ControlDirectoryName+LockArtifactName+OwnerMetadataName, `:*?"<>|`) { + t.Fatal("coordination artifact names are not Windows-safe") + } +} + +func TestValidateIdentityRejectsTampering(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + identity.Hash = strings.Repeat("0", sha256HexLength) + if err := ValidateIdentity(identity); !errors.Is(err, ErrRepositoryIdentityInvalid) { + t.Fatalf("expected invalid identity, got %v", err) + } +} diff --git a/internal/coordination/metadata.go b/internal/coordination/metadata.go new file mode 100644 index 00000000..648a35c6 --- /dev/null +++ b/internal/coordination/metadata.go @@ -0,0 +1,185 @@ +package coordination + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +const ( + ownerMetadataTempPattern = ".owner-*.tmp" + maxOwnerMetadataSize = 64 * 1024 +) + +// publishOwnerMetadata publishes diagnostic owner metadata as a complete, +// validated record. The metadata is not repository ownership proof. +func publishOwnerMetadata(prepared PreparedControlNamespace, owner Owner) (err error) { + if err := validatePreparedControlNamespace(prepared); err != nil { + return err + } + data, err := EncodeOwner(owner) + if err != nil { + return err + } + if owner.IdentityHash != prepared.Identity.Hash { + return fmt.Errorf("coordination: owner identity does not match prepared control namespace") + } + + temp, err := os.CreateTemp(prepared.ControlDirectory, ownerMetadataTempPattern) + if err != nil { + return fmt.Errorf("coordination: create owner metadata temporary file: %w", err) + } + tempPath := temp.Name() + removeTemp := true + defer func() { + if temp != nil { + if closeErr := temp.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("coordination: close owner metadata temporary file: %w", closeErr)) + } + } + if removeTemp { + if removeErr := os.Remove(tempPath); removeErr != nil && !os.IsNotExist(removeErr) { + err = errors.Join(err, fmt.Errorf("coordination: remove owner metadata temporary file: %w", removeErr)) + } + } + }() + + closed, err := writeSyncedOwnerMetadataTemp(temp, data) + if closed { + temp = nil + } + if err != nil { + return err + } + + if err := replaceOwnerMetadata(tempPath, prepared.OwnerMetadataPath); err != nil { + return err + } + removeTemp = false + return nil +} + +func writeSyncedOwnerMetadataTemp(temp *os.File, data []byte) (bool, error) { + written, err := temp.Write(data) + if err != nil { + return false, fmt.Errorf("coordination: write owner metadata temporary file: %w", err) + } + if written != len(data) { + return false, fmt.Errorf("coordination: write owner metadata temporary file: %w", io.ErrShortWrite) + } + if err := temp.Sync(); err != nil { + return false, fmt.Errorf("coordination: sync owner metadata temporary file: %w", err) + } + if err := temp.Close(); err != nil { + return true, fmt.Errorf("coordination: close owner metadata temporary file: %w", err) + } + return true, nil +} + +// readOwnerMetadata reads non-authoritative diagnostics. Missing, malformed, +// and unsupported metadata remain ordinary diagnostic errors, never Busy. +func readOwnerMetadata(prepared PreparedControlNamespace) (Owner, error) { + if err := validatePreparedControlNamespace(prepared); err != nil { + return Owner{}, err + } + data, err := readBoundedRegularOwnerMetadata(prepared.OwnerMetadataPath) + if err != nil { + return Owner{}, err + } + return DecodeOwner(data) +} + +func readBoundedRegularOwnerMetadata(path string) ([]byte, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("coordination: inspect owner metadata: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return nil, fmt.Errorf("coordination: owner metadata must be a regular file") + } + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("coordination: open owner metadata: %w", err) + } + data, readErr := io.ReadAll(io.LimitReader(file, int64(maxOwnerMetadataSize)+1)) + closeErr := file.Close() + if readErr != nil { + return nil, fmt.Errorf("coordination: read owner metadata: %w", readErr) + } + if closeErr != nil { + return nil, fmt.Errorf("coordination: close owner metadata: %w", closeErr) + } + if len(data) > maxOwnerMetadataSize { + return nil, fmt.Errorf("coordination: owner metadata exceeds maximum size of %d bytes", maxOwnerMetadataSize) + } + return data, nil +} + +func inspectOwnerMetadataDestination(path string) (bool, error) { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("coordination: inspect existing owner metadata: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return false, fmt.Errorf("coordination: existing owner metadata must be a regular file") + } + return true, nil +} + +// removeOwnerMetadata removes only the diagnostic owner record. Absence is a +// successful idempotent outcome; the persistent lock artifact is untouched. +func removeOwnerMetadata(prepared PreparedControlNamespace) error { + if err := validatePreparedControlNamespace(prepared); err != nil { + return err + } + info, err := os.Lstat(prepared.OwnerMetadataPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("coordination: inspect owner metadata for removal: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return fmt.Errorf("coordination: owner metadata must be a regular file") + } + if err := os.Remove(prepared.OwnerMetadataPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("coordination: remove owner metadata: %w", err) + } + return nil +} + +func validatePreparedControlNamespace(prepared PreparedControlNamespace) error { + if err := ValidateIdentity(prepared.Identity); err != nil { + return err + } + controlDirectory, err := ControlDirectory(prepared.Identity) + if err != nil { + return err + } + if !preparedNamespacePathsMatch(prepared, controlDirectory) { + return fmt.Errorf("%w: prepared control namespace paths do not match identity", ErrRepositoryIdentityInvalid) + } + return validateRealControlDirectory(controlDirectory) +} + +func preparedNamespacePathsMatch(prepared PreparedControlNamespace, controlDirectory string) bool { + return prepared.ControlDirectory == controlDirectory && + prepared.LockArtifactPath == filepath.Join(controlDirectory, LockArtifactName) && + prepared.OwnerMetadataPath == filepath.Join(controlDirectory, OwnerMetadataName) +} + +func validateRealControlDirectory(controlDirectory string) error { + info, err := os.Lstat(controlDirectory) + if err != nil { + return fmt.Errorf("%w: inspect prepared control directory: %w", ErrRepositoryIdentityInvalid, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("%w: prepared control path must be a real directory", ErrRepositoryIdentityInvalid) + } + return nil +} diff --git a/internal/coordination/metadata_replace_unix.go b/internal/coordination/metadata_replace_unix.go new file mode 100644 index 00000000..27dfaff7 --- /dev/null +++ b/internal/coordination/metadata_replace_unix.go @@ -0,0 +1,18 @@ +//go:build !windows + +package coordination + +import ( + "fmt" + "os" +) + +func replaceOwnerMetadata(tempPath, ownerPath string) error { + if _, err := inspectOwnerMetadataDestination(ownerPath); err != nil { + return err + } + if err := os.Rename(tempPath, ownerPath); err != nil { + return fmt.Errorf("coordination: publish complete owner metadata: %w", err) + } + return nil +} diff --git a/internal/coordination/metadata_replace_windows.go b/internal/coordination/metadata_replace_windows.go new file mode 100644 index 00000000..b926a33d --- /dev/null +++ b/internal/coordination/metadata_replace_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package coordination + +import ( + "fmt" + "os" +) + +func replaceOwnerMetadata(tempPath, ownerPath string) error { + exists, err := inspectOwnerMetadataDestination(ownerPath) + if err != nil { + return err + } + if exists { + if err := os.Remove(ownerPath); err != nil { + return fmt.Errorf("coordination: remove existing owner metadata before publication: %w", err) + } + } + if err := os.Rename(tempPath, ownerPath); err != nil { + return fmt.Errorf("coordination: publish complete owner metadata: %w", err) + } + return nil +} diff --git a/internal/coordination/metadata_test.go b/internal/coordination/metadata_test.go new file mode 100644 index 00000000..d982af25 --- /dev/null +++ b/internal/coordination/metadata_test.go @@ -0,0 +1,321 @@ +package coordination + +import ( + "bytes" + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +func TestOwnerMetadataPublicationReadAndSensitiveFieldBoundary(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + owner := mustMetadataOwner(t, prepared.Identity, OperationSnapshotCreate, testOwnerStart) + + if err := publishOwnerMetadata(prepared, owner); err != nil { + t.Fatalf("publishOwnerMetadata: %v", err) + } + raw, err := os.ReadFile(prepared.OwnerMetadataPath) + if err != nil { + t.Fatalf("read published owner metadata: %v", err) + } + if !json.Valid(raw) { + t.Fatalf("published owner metadata is not complete JSON: %s", raw) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatalf("decode owner metadata fields: %v", err) + } + allowed := map[string]bool{ + "schema_version": true, + "pid": true, + "operation": true, + "started_at": true, + "hostname": true, + "executable": true, + "version": true, + "identity_hash": true, + "mode": true, + } + for field := range fields { + if !allowed[field] { + t.Fatalf("published owner metadata contains unexpected field %q: %s", field, raw) + } + } + for _, field := range []string{"schema_version", "pid", "operation", "started_at", "version", "identity_hash", "mode"} { + if _, exists := fields[field]; !exists { + t.Fatalf("published owner metadata is missing field %q: %s", field, raw) + } + } + for _, forbidden := range []string{ + prepared.Identity.CanonicalPath, + "configured-secret-repository", + "postgres://user:password@database.example/coldkeep", + "secret-user-name", + "--restore-destination=/private/output", + "source_path", + "working_directory", + } { + if bytes.Contains(raw, []byte(forbidden)) { + t.Fatalf("published owner metadata contains sensitive value %q: %s", forbidden, raw) + } + } + + read, err := readOwnerMetadata(prepared) + if err != nil { + t.Fatalf("readOwnerMetadata: %v", err) + } + assertOwnerEqual(t, read, owner) + if runtime.GOOS != "windows" { + info, err := os.Stat(prepared.OwnerMetadataPath) + if err != nil { + t.Fatalf("stat owner metadata: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("owner metadata mode=%#o want=0600", info.Mode().Perm()) + } + } + assertOnlyOwnerMetadataArtifact(t, prepared) +} + +func TestOwnerMetadataPublicationReplacesCompleteRecord(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + first := mustMetadataOwner(t, prepared.Identity, OperationStore, testOwnerStart) + second := mustMetadataOwner(t, prepared.Identity, OperationVerify, testOwnerStart.Add(time.Minute)) + + if err := publishOwnerMetadata(prepared, first); err != nil { + t.Fatalf("publish first owner: %v", err) + } + if err := publishOwnerMetadata(prepared, second); err != nil { + t.Fatalf("publish second owner: %v", err) + } + read, err := readOwnerMetadata(prepared) + if err != nil { + t.Fatalf("read replacement owner: %v", err) + } + assertOwnerEqual(t, read, second) + assertOnlyOwnerMetadataArtifact(t, prepared) +} + +func TestOwnerMetadataPublicationRejectsMismatchedIdentity(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + otherIdentity := mustIdentity(t, t.TempDir()) + owner := mustMetadataOwner(t, otherIdentity, OperationStore, testOwnerStart) + + if err := publishOwnerMetadata(prepared, owner); err == nil { + t.Fatal("expected owner identity mismatch") + } + if _, err := os.Lstat(prepared.OwnerMetadataPath); !os.IsNotExist(err) { + t.Fatalf("mismatched owner metadata was published, stat err=%v", err) + } +} + +func TestOwnerMetadataPublicationRejectsSymlinkDestination(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + owner := mustMetadataOwner(t, prepared.Identity, OperationStore, testOwnerStart) + outsidePath := filepath.Join(t.TempDir(), "outside-owner.json") + outsideData := []byte("outside target must remain unchanged") + if err := os.WriteFile(outsidePath, outsideData, 0o600); err != nil { + t.Fatalf("write outside target: %v", err) + } + if err := os.Symlink(outsidePath, prepared.OwnerMetadataPath); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := publishOwnerMetadata(prepared, owner) + if err == nil { + t.Fatal("expected symlink destination rejection") + } + assertMetadataErrorIsNotOwnership(t, err) + info, statErr := os.Lstat(prepared.OwnerMetadataPath) + if statErr != nil { + t.Fatalf("lstat rejected owner metadata symlink: %v", statErr) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("owner metadata destination mode=%v want symlink", info.Mode()) + } + gotOutside, readErr := os.ReadFile(outsidePath) + if readErr != nil { + t.Fatalf("read outside target: %v", readErr) + } + if !bytes.Equal(gotOutside, outsideData) { + t.Fatalf("outside target changed: got=%q want=%q", gotOutside, outsideData) + } + assertOnlyOwnerMetadataArtifact(t, prepared) +} + +func TestOwnerMetadataPublicationRejectsDirectoryDestination(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + owner := mustMetadataOwner(t, prepared.Identity, OperationStore, testOwnerStart) + if err := os.Mkdir(prepared.OwnerMetadataPath, 0o700); err != nil { + t.Fatalf("create owner metadata directory: %v", err) + } + + err := publishOwnerMetadata(prepared, owner) + if err == nil { + t.Fatal("expected directory destination rejection") + } + assertMetadataErrorIsNotOwnership(t, err) + info, statErr := os.Lstat(prepared.OwnerMetadataPath) + if statErr != nil { + t.Fatalf("lstat rejected owner metadata directory: %v", statErr) + } + if !info.IsDir() { + t.Fatalf("owner metadata destination mode=%v want directory", info.Mode()) + } + assertOnlyOwnerMetadataArtifact(t, prepared) +} + +func TestOwnerMetadataReadMissingIsNonAuthoritative(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + _, err := readOwnerMetadata(prepared) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("read missing metadata error=%v want fs.ErrNotExist", err) + } + assertMetadataErrorIsNotOwnership(t, err) +} + +func TestOwnerMetadataReadRejectsMalformedAndUnsupportedData(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + for name, data := range map[string][]byte{ + "malformed": []byte(`{"schema_version":`), + "unsupported-schema": validOwnerWithSchemaVersion(t, prepared.Identity, 2), + } { + t.Run(name, func(t *testing.T) { + if err := os.WriteFile(prepared.OwnerMetadataPath, data, 0o600); err != nil { + t.Fatalf("write owner metadata fixture: %v", err) + } + if _, err := readOwnerMetadata(prepared); err == nil { + t.Fatal("expected diagnostic metadata error") + } else { + assertMetadataErrorIsNotOwnership(t, err) + } + }) + } +} + +func TestOwnerMetadataReadRejectsOversizedRegularFile(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + oversized := bytes.Repeat([]byte{'x'}, maxOwnerMetadataSize+1) + if err := os.WriteFile(prepared.OwnerMetadataPath, oversized, 0o600); err != nil { + t.Fatalf("write oversized owner metadata: %v", err) + } + + _, err := readOwnerMetadata(prepared) + if err == nil { + t.Fatal("expected oversized owner metadata error") + } + assertMetadataErrorIsNotOwnership(t, err) + info, statErr := os.Lstat(prepared.OwnerMetadataPath) + if statErr != nil { + t.Fatalf("oversized owner metadata was removed: %v", statErr) + } + if !info.Mode().IsRegular() { + t.Fatalf("oversized owner metadata mode=%v want regular", info.Mode()) + } +} + +func TestOwnerMetadataRemovalExistingAndAbsent(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + owner := mustMetadataOwner(t, prepared.Identity, OperationGarbageCollect, testOwnerStart) + if err := publishOwnerMetadata(prepared, owner); err != nil { + t.Fatalf("publish owner: %v", err) + } + + if err := removeOwnerMetadata(prepared); err != nil { + t.Fatalf("remove existing owner metadata: %v", err) + } + if _, err := os.Lstat(prepared.OwnerMetadataPath); !os.IsNotExist(err) { + t.Fatalf("owner metadata remains after removal, stat err=%v", err) + } + if err := removeOwnerMetadata(prepared); err != nil { + t.Fatalf("remove absent owner metadata: %v", err) + } + if _, err := os.Lstat(prepared.LockArtifactPath); !os.IsNotExist(err) { + t.Fatalf("metadata removal touched lock artifact, stat err=%v", err) + } +} + +func mustPreparedControlNamespace(t *testing.T) PreparedControlNamespace { + t.Helper() + prepared, err := PrepareControlNamespace(filepath.Join(t.TempDir(), "containers")) + if err != nil { + t.Fatalf("PrepareControlNamespace: %v", err) + } + return prepared +} + +func mustMetadataOwner(t *testing.T, identity Identity, operation Operation, startedAt time.Time) Owner { + t.Helper() + owner, err := NewOwner(operation, identity, "1.13.11", startedAt) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + owner.Hostname = "host.example" + owner.Executable = "coldkeep" + return owner +} + +func assertOwnerEqual(t *testing.T, got, want Owner) { + t.Helper() + if got.SchemaVersion != want.SchemaVersion || + got.PID != want.PID || + got.Operation != want.Operation || + !got.StartedAt.Equal(want.StartedAt) || + got.Hostname != want.Hostname || + got.Executable != want.Executable || + got.Version != want.Version || + got.IdentityHash != want.IdentityHash || + got.Mode != want.Mode { + t.Fatalf("owner mismatch got=%+v want=%+v", got, want) + } +} + +func assertOnlyOwnerMetadataArtifact(t *testing.T, prepared PreparedControlNamespace) { + t.Helper() + entries, err := os.ReadDir(prepared.ControlDirectory) + if err != nil { + t.Fatalf("read control directory: %v", err) + } + if len(entries) != 1 || entries[0].Name() != OwnerMetadataName { + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + t.Fatalf("unexpected control artifacts after publication: %s", strings.Join(names, ", ")) + } +} + +func assertMetadataErrorIsNotOwnership(t *testing.T, err error) { + t.Helper() + for _, sentinel := range []error{ + ErrRepositoryBusy, + ErrRepositoryLockUnsupported, + ErrRepositoryIdentityInvalid, + ErrNestedRepositoryAcquisition, + } { + if errors.Is(err, sentinel) { + t.Fatalf("diagnostic metadata error impersonates ownership error %v: %v", sentinel, err) + } + } +} + +func validOwnerWithSchemaVersion(t *testing.T, identity Identity, version int) []byte { + t.Helper() + owner := mustMetadataOwner(t, identity, OperationRestore, testOwnerStart) + encoded, err := EncodeOwner(owner) + if err != nil { + t.Fatalf("EncodeOwner: %v", err) + } + updated := strings.Replace(string(encoded), `"schema_version":1`, `"schema_version":`+strconv.Itoa(version), 1) + if updated == string(encoded) { + t.Fatal("owner schema fixture did not change") + } + return []byte(updated) +} diff --git a/internal/coordination/native_lock.go b/internal/coordination/native_lock.go new file mode 100644 index 00000000..672b0ef6 --- /dev/null +++ b/internal/coordination/native_lock.go @@ -0,0 +1,30 @@ +package coordination + +import "sync" + +// nativeLockHandle retains the platform lock resource until release. It is a +// low-level primitive and does not include process reservation or owner data. +type nativeLockHandle struct { + releaseOnce sync.Once + releaseFn func() error + releaseErr error +} + +func acquireNativeLock(prepared PreparedControlNamespace) (*nativeLockHandle, error) { + if err := validatePreparedControlNamespace(prepared); err != nil { + return nil, err + } + return acquireNativeLockPlatform(prepared.LockArtifactPath) +} + +func (handle *nativeLockHandle) release() error { + if handle == nil { + return nil + } + handle.releaseOnce.Do(func() { + if handle.releaseFn != nil { + handle.releaseErr = handle.releaseFn() + } + }) + return handle.releaseErr +} diff --git a/internal/coordination/native_lock_unix.go b/internal/coordination/native_lock_unix.go new file mode 100644 index 00000000..e0696fe6 --- /dev/null +++ b/internal/coordination/native_lock_unix.go @@ -0,0 +1,91 @@ +//go:build linux || darwin + +package coordination + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +func acquireNativeLockPlatform(lockPath string) (*nativeLockHandle, error) { + if err := inspectNativeLockArtifact(lockPath); err != nil { + return nil, err + } + + fd, err := unix.Open(lockPath, unix.O_RDWR|unix.O_CREAT|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + return nil, fmt.Errorf("coordination: open native repository lock: %w", err) + } + file := os.NewFile(uintptr(fd), LockArtifactName) + if file == nil { + primary := fmt.Errorf("coordination: create native repository lock file handle") + if closeErr := unix.Close(fd); closeErr != nil { + primary = errors.Join(primary, fmt.Errorf("coordination: close native repository lock descriptor: %w", closeErr)) + } + return nil, primary + } + + info, err := file.Stat() + if err != nil { + return nil, closeNativeLockFileAfterError(file, fmt.Errorf("coordination: inspect opened native repository lock: %w", err)) + } + if !info.Mode().IsRegular() { + return nil, closeNativeLockFileAfterError(file, fmt.Errorf("coordination: native repository lock must be a regular file")) + } + + if err := unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB); err != nil { + return nil, closeNativeLockFileAfterError(file, mapNativeFlockError(err)) + } + + return &nativeLockHandle{ + releaseFn: func() error { + return releaseNativeLockFile(file, fd) + }, + }, nil +} + +func inspectNativeLockArtifact(lockPath string) error { + info, err := os.Lstat(lockPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("coordination: inspect native repository lock: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return fmt.Errorf("coordination: native repository lock must be a regular file") + } + return nil +} + +func mapNativeFlockError(err error) error { + switch { + case errors.Is(err, unix.EWOULDBLOCK), errors.Is(err, unix.EAGAIN): + return fmt.Errorf("%w: native flock: %w", ErrRepositoryBusy, err) + case errors.Is(err, unix.ENOSYS), errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP): + return fmt.Errorf("%w: native flock: %w", ErrRepositoryLockUnsupported, err) + default: + return fmt.Errorf("coordination: acquire native repository lock: %w", err) + } +} + +func closeNativeLockFileAfterError(file *os.File, primary error) error { + if closeErr := file.Close(); closeErr != nil { + return errors.Join(primary, fmt.Errorf("coordination: close native repository lock after failed acquisition: %w", closeErr)) + } + return primary +} + +func releaseNativeLockFile(file *os.File, fd int) error { + var releaseErr error + if err := unix.Flock(fd, unix.LOCK_UN); err != nil { + releaseErr = fmt.Errorf("coordination: unlock native repository lock: %w", err) + } + if err := file.Close(); err != nil { + releaseErr = errors.Join(releaseErr, fmt.Errorf("coordination: close native repository lock: %w", err)) + } + return releaseErr +} diff --git a/internal/coordination/native_lock_unix_test.go b/internal/coordination/native_lock_unix_test.go new file mode 100644 index 00000000..28b23b31 --- /dev/null +++ b/internal/coordination/native_lock_unix_test.go @@ -0,0 +1,280 @@ +//go:build linux || darwin + +package coordination + +import ( + "bytes" + "errors" + "os" + "sync" + "testing" + + "golang.org/x/sys/unix" +) + +func TestNativeLockCreatesPersistentArtifact(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + if _, err := os.Lstat(prepared.LockArtifactPath); !os.IsNotExist(err) { + t.Fatalf("lock artifact exists before acquisition, stat err=%v", err) + } + + handle := mustAcquireNativeLock(t, prepared) + info, err := os.Lstat(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("lstat acquired lock artifact: %v", err) + } + if !info.Mode().IsRegular() { + t.Fatalf("lock artifact mode=%v want regular", info.Mode()) + } + if mode := info.Mode().Perm(); mode&^os.FileMode(0o600) != 0 { + t.Fatalf("new lock artifact mode=%#o exceeds requested 0600", mode) + } + if _, err := os.Lstat(prepared.OwnerMetadataPath); !os.IsNotExist(err) { + t.Fatalf("native acquisition created owner metadata, stat err=%v", err) + } + + if err := handle.release(); err != nil { + t.Fatalf("release native lock: %v", err) + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("persistent lock artifact missing after release: %v", err) + } else if !info.Mode().IsRegular() { + t.Fatalf("persistent lock artifact mode=%v want regular", info.Mode()) + } +} + +func TestNativeLockPreservesExistingPermissionsAndContents(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + wantContents := []byte("persistent lock artifact contents\n") + if err := os.WriteFile(prepared.LockArtifactPath, wantContents, 0o600); err != nil { + t.Fatalf("write existing lock artifact: %v", err) + } + if err := os.Chmod(prepared.LockArtifactPath, 0o640); err != nil { + t.Fatalf("set existing lock artifact mode: %v", err) + } + + handle := mustAcquireNativeLock(t, prepared) + if err := handle.release(); err != nil { + t.Fatalf("release native lock: %v", err) + } + + info, err := os.Stat(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("stat existing lock artifact: %v", err) + } + if mode := info.Mode().Perm(); mode != 0o640 { + t.Fatalf("existing lock artifact mode=%#o want=0640", mode) + } + gotContents, err := os.ReadFile(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("read existing lock artifact: %v", err) + } + if !bytes.Equal(gotContents, wantContents) { + t.Fatalf("existing lock artifact contents=%q want=%q", gotContents, wantContents) + } +} + +func TestNativeLockRejectsUnsafeArtifacts(t *testing.T) { + t.Run("symlink", func(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + outsidePath := t.TempDir() + "/outside-lock" + outsideContents := []byte("outside target remains unchanged") + if err := os.WriteFile(outsidePath, outsideContents, 0o600); err != nil { + t.Fatalf("write outside target: %v", err) + } + if err := os.Symlink(outsidePath, prepared.LockArtifactPath); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + if handle, err := acquireNativeLock(prepared); err == nil { + _ = handle.release() + t.Fatal("expected symlink lock artifact rejection") + } + info, err := os.Lstat(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("lstat rejected symlink: %v", err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("lock artifact mode=%v want symlink", info.Mode()) + } + gotContents, err := os.ReadFile(outsidePath) + if err != nil { + t.Fatalf("read outside target: %v", err) + } + if !bytes.Equal(gotContents, outsideContents) { + t.Fatalf("outside target contents=%q want=%q", gotContents, outsideContents) + } + }) + + t.Run("directory", func(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + if err := os.Mkdir(prepared.LockArtifactPath, 0o700); err != nil { + t.Fatalf("create lock artifact directory: %v", err) + } + if handle, err := acquireNativeLock(prepared); err == nil { + _ = handle.release() + t.Fatal("expected directory lock artifact rejection") + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("lstat rejected directory: %v", err) + } else if !info.IsDir() { + t.Fatalf("lock artifact mode=%v want directory", info.Mode()) + } + }) + + t.Run("fifo", func(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + if err := unix.Mkfifo(prepared.LockArtifactPath, 0o600); err != nil { + t.Skipf("FIFO creation unavailable: %v", err) + } + if handle, err := acquireNativeLock(prepared); err == nil { + _ = handle.release() + t.Fatal("expected FIFO lock artifact rejection") + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("lstat rejected FIFO: %v", err) + } else if info.Mode()&os.ModeNamedPipe == 0 { + t.Fatalf("lock artifact mode=%v want FIFO", info.Mode()) + } + }) +} + +func TestNativeLockContentionAndReacquire(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + holder := mustAcquireNativeLock(t, prepared) + + contender, err := acquireNativeLock(prepared) + if contender != nil { + _ = contender.release() + t.Fatal("contending native acquisition returned a handle") + } + if !errors.Is(err, ErrRepositoryBusy) { + t.Fatalf("contending native acquisition error=%v want ErrRepositoryBusy", err) + } + + if err := holder.release(); err != nil { + t.Fatalf("release holder: %v", err) + } + reacquired := mustAcquireNativeLock(t, prepared) + if err := reacquired.release(); err != nil { + t.Fatalf("release reacquired native lock: %v", err) + } +} + +func TestNativeLockReleaseIsIdempotentAndCannotDamageSuccessor(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + first := mustAcquireNativeLock(t, prepared) + if err := first.release(); err != nil { + t.Fatalf("release first native lock: %v", err) + } + if err := first.release(); err != nil { + t.Fatalf("second release of first native lock: %v", err) + } + + successor := mustAcquireNativeLock(t, prepared) + if err := first.release(); err != nil { + t.Fatalf("stale release of first native lock: %v", err) + } + contender, err := acquireNativeLock(prepared) + if contender != nil { + _ = contender.release() + t.Fatal("stale release unlocked successor") + } + if !errors.Is(err, ErrRepositoryBusy) { + t.Fatalf("contention after stale release error=%v want ErrRepositoryBusy", err) + } + if err := successor.release(); err != nil { + t.Fatalf("release successor native lock: %v", err) + } +} + +func TestNativeLockAllowsDifferentRepositories(t *testing.T) { + firstPrepared := mustPreparedControlNamespace(t) + secondPrepared := mustPreparedControlNamespace(t) + first := mustAcquireNativeLock(t, firstPrepared) + second := mustAcquireNativeLock(t, secondPrepared) + if err := second.release(); err != nil { + t.Fatalf("release second repository lock: %v", err) + } + if err := first.release(); err != nil { + t.Fatalf("release first repository lock: %v", err) + } +} + +func TestNativeLockConcurrentContention(t *testing.T) { + const competitors = 32 + prepared := mustPreparedControlNamespace(t) + holder := mustAcquireNativeLock(t, prepared) + start := make(chan struct{}) + results := make(chan nativeLockResult, competitors) + var workers sync.WaitGroup + workers.Add(competitors) + for range competitors { + go func() { + defer workers.Done() + <-start + handle, err := acquireNativeLock(prepared) + results <- nativeLockResult{handle: handle, err: err} + }() + } + close(start) + workers.Wait() + close(results) + + for result := range results { + if result.handle != nil { + _ = result.handle.release() + t.Fatal("concurrent contender acquired held native lock") + } + if !errors.Is(result.err, ErrRepositoryBusy) { + t.Fatalf("concurrent contention error=%v want ErrRepositoryBusy", result.err) + } + } + if err := holder.release(); err != nil { + t.Fatalf("release contention holder: %v", err) + } + reacquired := mustAcquireNativeLock(t, prepared) + if err := reacquired.release(); err != nil { + t.Fatalf("release post-contention native lock: %v", err) + } +} + +func TestNativeFlockErrorMapping(t *testing.T) { + for name, test := range map[string]struct { + err error + want error + notWant error + }{ + "busy": {err: unix.EWOULDBLOCK, want: ErrRepositoryBusy, notWant: ErrRepositoryLockUnsupported}, + "unsupported": {err: unix.ENOSYS, want: ErrRepositoryLockUnsupported, notWant: ErrRepositoryBusy}, + "permission": {err: unix.EACCES, want: unix.EACCES, notWant: ErrRepositoryLockUnsupported}, + "unexpected": {err: unix.EIO, want: unix.EIO, notWant: ErrRepositoryLockUnsupported}, + } { + t.Run(name, func(t *testing.T) { + err := mapNativeFlockError(test.err) + if !errors.Is(err, test.want) { + t.Fatalf("mapped error=%v want errors.Is(%v)", err, test.want) + } + if errors.Is(err, test.notWant) { + t.Fatalf("mapped error=%v unexpectedly matches %v", err, test.notWant) + } + }) + } +} + +func mustAcquireNativeLock(t *testing.T, prepared PreparedControlNamespace) *nativeLockHandle { + t.Helper() + handle, err := acquireNativeLock(prepared) + if err != nil { + t.Fatalf("acquireNativeLock: %v", err) + } + if handle == nil { + t.Fatal("acquireNativeLock returned nil handle") + } + return handle +} + +type nativeLockResult struct { + handle *nativeLockHandle + err error +} diff --git a/internal/coordination/native_lock_unsupported.go b/internal/coordination/native_lock_unsupported.go new file mode 100644 index 00000000..04be795f --- /dev/null +++ b/internal/coordination/native_lock_unsupported.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin && !windows + +package coordination + +import "fmt" + +func acquireNativeLockPlatform(string) (*nativeLockHandle, error) { + return nil, fmt.Errorf("%w: native repository locking is unavailable on this platform", ErrRepositoryLockUnsupported) +} diff --git a/internal/coordination/native_lock_windows.go b/internal/coordination/native_lock_windows.go new file mode 100644 index 00000000..bafe57e3 --- /dev/null +++ b/internal/coordination/native_lock_windows.go @@ -0,0 +1,130 @@ +//go:build windows + +package coordination + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +const ( + windowsNativeLockBytesLow = 1 + windowsNativeLockBytesHigh = 0 +) + +func acquireNativeLockPlatform(lockPath string) (*nativeLockHandle, error) { + if err := inspectWindowsNativeLockArtifact(lockPath); err != nil { + return nil, err + } + + path, err := windows.UTF16PtrFromString(lockPath) + if err != nil { + return nil, fmt.Errorf("coordination: encode native repository lock path: %w", err) + } + handle, err := windows.CreateFile( + path, + windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + nil, + windows.OPEN_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, fmt.Errorf("coordination: open native repository lock: %w", err) + } + + if err := validateWindowsNativeLockHandle(handle); err != nil { + return nil, closeWindowsNativeLockHandleAfterError(handle, err) + } + + overlapped := &windows.Overlapped{} + if err := windows.LockFileEx( + handle, + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + windowsNativeLockBytesLow, + windowsNativeLockBytesHigh, + overlapped, + ); err != nil { + return nil, closeWindowsNativeLockHandleAfterError(handle, mapWindowsNativeLockError(err)) + } + + return &nativeLockHandle{ + releaseFn: func() error { + return releaseWindowsNativeLockHandle(handle) + }, + }, nil +} + +func inspectWindowsNativeLockArtifact(lockPath string) error { + info, err := os.Lstat(lockPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("coordination: inspect native repository lock: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return fmt.Errorf("coordination: native repository lock must be a regular file") + } + return nil +} + +func validateWindowsNativeLockHandle(handle windows.Handle) error { + fileType, err := windows.GetFileType(handle) + if err != nil { + return fmt.Errorf("coordination: inspect native repository lock type: %w", err) + } + if fileType != windows.FILE_TYPE_DISK { + return fmt.Errorf("coordination: native repository lock must be a disk file") + } + + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return fmt.Errorf("coordination: inspect native repository lock attributes: %w", err) + } + if info.FileAttributes&(windows.FILE_ATTRIBUTE_DIRECTORY|windows.FILE_ATTRIBUTE_REPARSE_POINT) != 0 { + return fmt.Errorf("coordination: native repository lock must be a regular non-reparse file") + } + return nil +} + +func mapWindowsNativeLockError(err error) error { + switch { + case errors.Is(err, windows.ERROR_LOCK_VIOLATION): + return fmt.Errorf("%w: native LockFileEx: %w", ErrRepositoryBusy, err) + case errors.Is(err, windows.ERROR_NOT_SUPPORTED): + return fmt.Errorf("%w: native LockFileEx: %w", ErrRepositoryLockUnsupported, err) + default: + return fmt.Errorf("coordination: acquire native repository lock: %w", err) + } +} + +func closeWindowsNativeLockHandleAfterError(handle windows.Handle, primary error) error { + if closeErr := windows.CloseHandle(handle); closeErr != nil { + return errors.Join(primary, fmt.Errorf("coordination: close native repository lock after failed acquisition: %w", closeErr)) + } + return primary +} + +func releaseWindowsNativeLockHandle(handle windows.Handle) error { + var releaseErr error + overlapped := &windows.Overlapped{} + if err := windows.UnlockFileEx( + handle, + 0, + windowsNativeLockBytesLow, + windowsNativeLockBytesHigh, + overlapped, + ); err != nil { + releaseErr = fmt.Errorf("coordination: unlock native repository lock: %w", err) + } + if err := windows.CloseHandle(handle); err != nil { + releaseErr = errors.Join(releaseErr, fmt.Errorf("coordination: close native repository lock: %w", err)) + } + return releaseErr +} diff --git a/internal/coordination/native_lock_windows_test.go b/internal/coordination/native_lock_windows_test.go new file mode 100644 index 00000000..a66dda16 --- /dev/null +++ b/internal/coordination/native_lock_windows_test.go @@ -0,0 +1,299 @@ +//go:build windows + +package coordination + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "sync" + "testing" + + "golang.org/x/sys/windows" +) + +func TestWindowsNativeLockCreatesPersistentArtifact(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + if _, err := os.Lstat(prepared.LockArtifactPath); !os.IsNotExist(err) { + t.Fatalf("lock artifact exists before acquisition, stat err=%v", err) + } + + handle := mustAcquireWindowsNativeLock(t, prepared) + info, err := os.Lstat(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("lstat acquired lock artifact: %v", err) + } + if !info.Mode().IsRegular() { + t.Fatalf("lock artifact mode=%v want regular", info.Mode()) + } + if err := handle.release(); err != nil { + t.Fatalf("release native lock: %v", err) + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("persistent lock artifact missing after release: %v", err) + } else if !info.Mode().IsRegular() { + t.Fatalf("persistent lock artifact mode=%v want regular", info.Mode()) + } +} + +func TestWindowsNativeLockPreservesExistingContents(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + wantContents := []byte("persistent Windows lock artifact contents\r\n") + if err := os.WriteFile(prepared.LockArtifactPath, wantContents, 0o600); err != nil { + t.Fatalf("write existing lock artifact: %v", err) + } + + handle := mustAcquireWindowsNativeLock(t, prepared) + if err := handle.release(); err != nil { + t.Fatalf("release native lock: %v", err) + } + gotContents, err := os.ReadFile(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("read existing lock artifact: %v", err) + } + if !bytes.Equal(gotContents, wantContents) { + t.Fatalf("existing lock artifact contents=%q want=%q", gotContents, wantContents) + } +} + +func TestWindowsNativeLockPreservesEmptyArtifactLength(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + if err := os.WriteFile(prepared.LockArtifactPath, nil, 0o600); err != nil { + t.Fatalf("create empty lock artifact: %v", err) + } + + handle := mustAcquireWindowsNativeLock(t, prepared) + if err := handle.release(); err != nil { + t.Fatalf("release native lock: %v", err) + } + info, err := os.Stat(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("stat empty lock artifact: %v", err) + } + if info.Size() != 0 { + t.Fatalf("empty lock artifact size=%d want=0", info.Size()) + } +} + +func TestWindowsNativeLockRejectsDirectoryArtifact(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + if err := os.Mkdir(prepared.LockArtifactPath, 0o700); err != nil { + t.Fatalf("create lock artifact directory: %v", err) + } + if handle, err := acquireNativeLock(prepared); err == nil { + _ = handle.release() + t.Fatal("expected directory lock artifact rejection") + } + if info, err := os.Lstat(prepared.LockArtifactPath); err != nil { + t.Fatalf("lstat rejected directory: %v", err) + } else if !info.IsDir() { + t.Fatalf("lock artifact mode=%v want directory", info.Mode()) + } +} + +func TestWindowsNativeLockRejectsSymlinkArtifact(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + outsidePath := filepath.Join(t.TempDir(), "outside-lock") + outsideContents := []byte("outside target remains unchanged") + if err := os.WriteFile(outsidePath, outsideContents, 0o600); err != nil { + t.Fatalf("write outside target: %v", err) + } + if err := os.Symlink(outsidePath, prepared.LockArtifactPath); err != nil { + t.Skipf("Windows file symlink creation unavailable: %v", err) + } + + if handle, err := acquireNativeLock(prepared); err == nil { + _ = handle.release() + t.Fatal("expected symlink lock artifact rejection") + } + info, err := os.Lstat(prepared.LockArtifactPath) + if err != nil { + t.Fatalf("lstat rejected symlink: %v", err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("lock artifact mode=%v want symlink", info.Mode()) + } + gotContents, err := os.ReadFile(outsidePath) + if err != nil { + t.Fatalf("read outside target: %v", err) + } + if !bytes.Equal(gotContents, outsideContents) { + t.Fatalf("outside target contents=%q want=%q", gotContents, outsideContents) + } +} + +func TestWindowsNativeLockContentionAndReacquire(t *testing.T) { + prepared := mustPreparedControlNamespace(t) + holder := mustAcquireWindowsNativeLock(t, prepared) + + contender, err := acquireNativeLock(prepared) + if contender != nil { + _ = contender.release() + t.Fatal("contending native acquisition returned a handle") + } + if !errors.Is(err, ErrRepositoryBusy) { + t.Fatalf("contending native acquisition error=%v want ErrRepositoryBusy", err) + } + if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + t.Fatalf("contending native acquisition error=%v want ERROR_LOCK_VIOLATION", err) + } + + if err := holder.release(); err != nil { + t.Fatalf("release holder: %v", err) + } + reacquired := mustAcquireWindowsNativeLock(t, prepared) + if err := reacquired.release(); err != nil { + t.Fatalf("release reacquired native lock: %v", err) + } +} + +func TestWindowsNativeLockReleaseIsIdempotentAndCannotDamageSuccessor(t *testing.T) { + const releasers = 32 + prepared := mustPreparedControlNamespace(t) + first := mustAcquireWindowsNativeLock(t, prepared) + errorsByRelease := make(chan error, releasers) + var workers sync.WaitGroup + workers.Add(releasers) + for range releasers { + go func() { + defer workers.Done() + errorsByRelease <- first.release() + }() + } + workers.Wait() + close(errorsByRelease) + for err := range errorsByRelease { + if err != nil { + t.Fatalf("concurrent release: %v", err) + } + } + + successor := mustAcquireWindowsNativeLock(t, prepared) + if err := first.release(); err != nil { + t.Fatalf("stale release of first native lock: %v", err) + } + contender, err := acquireNativeLock(prepared) + if contender != nil { + _ = contender.release() + t.Fatal("stale release unlocked successor") + } + if !errors.Is(err, ErrRepositoryBusy) { + t.Fatalf("contention after stale release error=%v want ErrRepositoryBusy", err) + } + if err := successor.release(); err != nil { + t.Fatalf("release successor native lock: %v", err) + } +} + +func TestWindowsNativeLockAllowsDifferentRepositories(t *testing.T) { + firstPrepared := mustPreparedControlNamespace(t) + secondPrepared := mustPreparedControlNamespace(t) + first := mustAcquireWindowsNativeLock(t, firstPrepared) + second := mustAcquireWindowsNativeLock(t, secondPrepared) + if err := second.release(); err != nil { + t.Fatalf("release second repository lock: %v", err) + } + if err := first.release(); err != nil { + t.Fatalf("release first repository lock: %v", err) + } +} + +func TestWindowsNativeLockConcurrentContention(t *testing.T) { + const competitors = 32 + prepared := mustPreparedControlNamespace(t) + holder := mustAcquireWindowsNativeLock(t, prepared) + start := make(chan struct{}) + results := make(chan windowsNativeLockResult, competitors) + var workers sync.WaitGroup + workers.Add(competitors) + for range competitors { + go func() { + defer workers.Done() + <-start + handle, err := acquireNativeLock(prepared) + results <- windowsNativeLockResult{handle: handle, err: err} + }() + } + close(start) + workers.Wait() + close(results) + + for result := range results { + if result.handle != nil { + _ = result.handle.release() + t.Fatal("concurrent contender acquired held native lock") + } + if !errors.Is(result.err, ErrRepositoryBusy) { + t.Fatalf("concurrent contention error=%v want ErrRepositoryBusy", result.err) + } + if !errors.Is(result.err, windows.ERROR_LOCK_VIOLATION) { + t.Fatalf("concurrent contention error=%v want ERROR_LOCK_VIOLATION", result.err) + } + } + if err := holder.release(); err != nil { + t.Fatalf("release contention holder: %v", err) + } + reacquired := mustAcquireWindowsNativeLock(t, prepared) + if err := reacquired.release(); err != nil { + t.Fatalf("release post-contention native lock: %v", err) + } +} + +func TestWindowsNativeLockErrorMapping(t *testing.T) { + for name, test := range map[string]struct { + err error + want error + notWant []error + }{ + "busy": { + err: windows.ERROR_LOCK_VIOLATION, want: ErrRepositoryBusy, + notWant: []error{ErrRepositoryLockUnsupported}, + }, + "unsupported": { + err: windows.ERROR_NOT_SUPPORTED, want: ErrRepositoryLockUnsupported, + notWant: []error{ErrRepositoryBusy}, + }, + "sharing": { + err: windows.ERROR_SHARING_VIOLATION, want: windows.ERROR_SHARING_VIOLATION, + notWant: []error{ErrRepositoryBusy, ErrRepositoryLockUnsupported}, + }, + "permission": { + err: windows.ERROR_ACCESS_DENIED, want: windows.ERROR_ACCESS_DENIED, + notWant: []error{ErrRepositoryBusy, ErrRepositoryLockUnsupported}, + }, + "unexpected": { + err: windows.ERROR_INVALID_DATA, want: windows.ERROR_INVALID_DATA, + notWant: []error{ErrRepositoryBusy, ErrRepositoryLockUnsupported}, + }, + } { + t.Run(name, func(t *testing.T) { + err := mapWindowsNativeLockError(test.err) + if !errors.Is(err, test.want) { + t.Fatalf("mapped error=%v want errors.Is(%v)", err, test.want) + } + for _, notWant := range test.notWant { + if errors.Is(err, notWant) { + t.Fatalf("mapped error=%v unexpectedly matches %v", err, notWant) + } + } + }) + } +} + +func mustAcquireWindowsNativeLock(t *testing.T, prepared PreparedControlNamespace) *nativeLockHandle { + t.Helper() + handle, err := acquireNativeLock(prepared) + if err != nil { + t.Fatalf("acquireNativeLock: %v", err) + } + if handle == nil { + t.Fatal("acquireNativeLock returned nil handle") + } + return handle +} + +type windowsNativeLockResult struct { + handle *nativeLockHandle + err error +} diff --git a/internal/coordination/owner.go b/internal/coordination/owner.go new file mode 100644 index 00000000..260e9197 --- /dev/null +++ b/internal/coordination/owner.go @@ -0,0 +1,141 @@ +package coordination + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +const OwnerMetadataSchemaVersion = 1 + +// Owner is non-authoritative diagnostic metadata for a native lease. +// It intentionally contains no repository path, DSN, credentials, user name, +// command arguments, source path, or destination path. +type Owner struct { + SchemaVersion int `json:"schema_version"` + PID int `json:"pid"` + Operation Operation `json:"operation"` + StartedAt time.Time `json:"started_at"` + Hostname string `json:"hostname,omitempty"` + Executable string `json:"executable,omitempty"` + Version string `json:"version"` + IdentityHash string `json:"identity_hash"` + Mode Mode `json:"mode"` +} + +// NewOwner builds the diagnostic record for an exclusive acquisition. +func NewOwner(operation Operation, identity Identity, version string, startedAt time.Time) (Owner, error) { + if err := ValidateIdentity(identity); err != nil { + return Owner{}, err + } + if !isCanonicalOperation(operation) { + return Owner{}, fmt.Errorf("coordination: unsupported owner operation %q", operation) + } + if strings.TrimSpace(version) == "" { + return Owner{}, fmt.Errorf("coordination: owner version is required") + } + if startedAt.IsZero() { + return Owner{}, fmt.Errorf("coordination: owner start time is required") + } + hostname, _ := os.Hostname() + return Owner{ + SchemaVersion: OwnerMetadataSchemaVersion, + PID: os.Getpid(), + Operation: operation, + StartedAt: startedAt.UTC(), + Hostname: strings.TrimSpace(hostname), + Executable: filepath.Base(os.Args[0]), + Version: strings.TrimSpace(version), + IdentityHash: identity.Hash, + Mode: ModeExclusive, + }, nil +} + +// EncodeOwner validates and deterministically serializes diagnostic metadata. +func EncodeOwner(owner Owner) ([]byte, error) { + if err := ValidateOwner(owner); err != nil { + return nil, err + } + owner.StartedAt = owner.StartedAt.UTC() + return json.Marshal(owner) +} + +// DecodeOwner strictly decodes one diagnostic metadata object. +func DecodeOwner(data []byte) (Owner, error) { + var owner Owner + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&owner); err != nil { + return Owner{}, fmt.Errorf("coordination: decode owner metadata: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return Owner{}, fmt.Errorf("coordination: decode owner metadata: trailing JSON value") + } + return Owner{}, fmt.Errorf("coordination: decode owner metadata: %w", err) + } + if err := ValidateOwner(owner); err != nil { + return Owner{}, err + } + owner.StartedAt = owner.StartedAt.UTC() + return owner, nil +} + +// ValidateOwner validates required, non-sensitive diagnostic fields. +func ValidateOwner(owner Owner) error { + if owner.SchemaVersion != OwnerMetadataSchemaVersion { + return fmt.Errorf("coordination: unsupported owner metadata schema %d", owner.SchemaVersion) + } + if owner.PID <= 0 { + return fmt.Errorf("coordination: owner PID must be positive") + } + if !isCanonicalOperation(owner.Operation) { + return fmt.Errorf("coordination: unsupported owner operation %q", owner.Operation) + } + if owner.StartedAt.IsZero() { + return fmt.Errorf("coordination: owner start time is required") + } + if strings.TrimSpace(owner.Version) == "" { + return fmt.Errorf("coordination: owner version is required") + } + if err := validateOwnerIdentity(owner); err != nil { + return err + } + return validateOwnerDisplayFields(owner) +} + +func validateOwnerIdentity(owner Owner) error { + if len(owner.IdentityHash) != sha256HexLength || !isLowerHex(owner.IdentityHash) { + return fmt.Errorf("coordination: owner identity hash must be lowercase SHA-256") + } + if owner.Mode != ModeExclusive { + return fmt.Errorf("coordination: owner mode must be %q", ModeExclusive) + } + return nil +} + +func validateOwnerDisplayFields(owner Owner) error { + if strings.ContainsAny(owner.Hostname, "\r\n") || strings.ContainsAny(owner.Executable, "\r\n") { + return fmt.Errorf("coordination: owner metadata contains a line break") + } + if owner.Executable != "" && filepath.Base(owner.Executable) != owner.Executable { + return fmt.Errorf("coordination: owner executable must be a basename") + } + return nil +} + +const sha256HexLength = 64 + +func isLowerHex(value string) bool { + for _, r := range value { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} diff --git a/internal/coordination/owner_test.go b/internal/coordination/owner_test.go new file mode 100644 index 00000000..b7e1ea19 --- /dev/null +++ b/internal/coordination/owner_test.go @@ -0,0 +1,140 @@ +package coordination + +import ( + "bytes" + "encoding/json" + "errors" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestOwnerMetadataRoundTripAndSensitiveFieldBoundary(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + startedAt := time.Date(2026, time.July, 25, 12, 30, 45, 123, time.FixedZone("test", 2*60*60)) + owner, err := NewOwner(OperationSnapshotRestore, identity, "1.13.11", startedAt) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + encoded, err := EncodeOwner(owner) + if err != nil { + t.Fatalf("EncodeOwner: %v", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(encoded, &fields); err != nil { + t.Fatalf("decode field map: %v", err) + } + for _, name := range []string{ + "schema_version", + "pid", + "operation", + "started_at", + "version", + "identity_hash", + "mode", + } { + if _, ok := fields[name]; !ok { + t.Fatalf("owner metadata missing stable field %q: %s", name, encoded) + } + } + if !bytes.Contains(encoded, []byte(`"started_at":"2026-07-25T10:30:45.000000123Z"`)) { + t.Fatalf("owner timestamp is not serialized in UTC: %s", encoded) + } + + for _, forbidden := range []string{ + identity.CanonicalPath, + "DB_PASSWORD", + "postgres://", + "source_path", + "destination_path", + "user_name", + } { + if forbidden != "" && bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("owner metadata contains forbidden value %q: %s", forbidden, encoded) + } + } + + decoded, err := DecodeOwner(encoded) + if err != nil { + t.Fatalf("DecodeOwner: %v", err) + } + if decoded.SchemaVersion != OwnerMetadataSchemaVersion || + decoded.PID != owner.PID || + decoded.Operation != OperationSnapshotRestore || + decoded.Mode != ModeExclusive || + decoded.Version != "1.13.11" || + decoded.IdentityHash != identity.Hash { + t.Fatalf("unexpected decoded owner: %+v", decoded) + } + if decoded.StartedAt.Location() != time.UTC || !decoded.StartedAt.Equal(startedAt) { + t.Fatalf("owner start time not normalized to UTC: %v", decoded.StartedAt) + } +} + +func TestOwnerMetadataOptionalFieldsAndExecutableBasename(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + owner, err := NewOwner(OperationVerify, identity, "1.13.11", testOwnerStart) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + owner.Hostname = "" + owner.Executable = "" + encoded, err := EncodeOwner(owner) + if err != nil { + t.Fatalf("EncodeOwner without optional fields: %v", err) + } + if bytes.Contains(encoded, []byte(`"hostname"`)) || bytes.Contains(encoded, []byte(`"executable"`)) { + t.Fatalf("empty optional fields were serialized: %s", encoded) + } + + owner.Executable = filepath.Join("private", "coldkeep") + if _, err := EncodeOwner(owner); err == nil { + t.Fatal("expected executable path to be rejected") + } +} + +func TestOwnerMetadataRejectsUnknownOperation(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + if _, err := NewOwner(Operation("unknown"), identity, "1.13.11", testOwnerStart); err == nil { + t.Fatal("expected unknown owner operation to fail") + } +} + +func TestOwnerMetadataRejectsMalformedAndUnknownFields(t *testing.T) { + identity := mustIdentity(t, t.TempDir()) + owner, err := NewOwner(OperationVerify, identity, "1.13.11", time.Now()) + if err != nil { + t.Fatalf("NewOwner: %v", err) + } + encoded, err := EncodeOwner(owner) + if err != nil { + t.Fatalf("EncodeOwner: %v", err) + } + + withUnknown := strings.TrimSuffix(string(encoded), "}") + `,"repository_path":"/secret"}` + if _, err := DecodeOwner([]byte(withUnknown)); err == nil { + t.Fatal("expected unknown owner metadata field to fail") + } + if _, err := DecodeOwner([]byte(`{"schema_version":1}`)); err == nil { + t.Fatal("expected incomplete owner metadata to fail") + } + unknownVersion := strings.Replace(string(encoded), `"schema_version":1`, `"schema_version":2`, 1) + if _, err := DecodeOwner([]byte(unknownVersion)); err == nil { + t.Fatal("expected unknown owner metadata version to fail") + } +} + +func TestCoordinationStableErrorSentinelsRemainDiscoverable(t *testing.T) { + for _, sentinel := range []error{ + ErrRepositoryBusy, + ErrRepositoryLockUnsupported, + ErrRepositoryIdentityInvalid, + ErrNestedRepositoryAcquisition, + } { + wrapped := errors.Join(errors.New("outer"), sentinel) + if !errors.Is(wrapped, sentinel) { + t.Fatalf("sentinel %v was not discoverable", sentinel) + } + } +} diff --git a/internal/coordination/registry.go b/internal/coordination/registry.go new file mode 100644 index 00000000..e40bcb21 --- /dev/null +++ b/internal/coordination/registry.go @@ -0,0 +1,53 @@ +package coordination + +import ( + "fmt" + "sync" +) + +// processRegistry reserves canonical repository identities within one process. +// It does not represent native repository ownership. +type processRegistry struct { + mu sync.Mutex + held map[string]*processReservation +} + +type processReservation struct { + registry *processRegistry + identityHash string + releaseOnce sync.Once +} + +func (registry *processRegistry) reserve(identity Identity) (*processReservation, error) { + if err := ValidateIdentity(identity); err != nil { + return nil, err + } + reservation := &processReservation{ + registry: registry, + identityHash: identity.Hash, + } + + registry.mu.Lock() + defer registry.mu.Unlock() + if registry.held == nil { + registry.held = make(map[string]*processReservation) + } + if _, exists := registry.held[identity.Hash]; exists { + return nil, fmt.Errorf("%w: repository identity is already reserved in this process", ErrNestedRepositoryAcquisition) + } + registry.held[identity.Hash] = reservation + return reservation, nil +} + +func (reservation *processReservation) release() { + if reservation == nil || reservation.registry == nil { + return + } + reservation.releaseOnce.Do(func() { + reservation.registry.mu.Lock() + defer reservation.registry.mu.Unlock() + if reservation.registry.held[reservation.identityHash] == reservation { + delete(reservation.registry.held, reservation.identityHash) + } + }) +} diff --git a/internal/coordination/registry_test.go b/internal/coordination/registry_test.go new file mode 100644 index 00000000..e9b0116b --- /dev/null +++ b/internal/coordination/registry_test.go @@ -0,0 +1,164 @@ +package coordination + +import ( + "errors" + "os" + "path/filepath" + "sync" + "testing" +) + +func TestProcessRegistryRejectsSequentialNestedReservationAndAllowsReacquire(t *testing.T) { + registry := &processRegistry{} + identity := mustIdentity(t, t.TempDir()) + reservation, err := registry.reserve(identity) + if err != nil { + t.Fatalf("reserve identity: %v", err) + } + if _, err := registry.reserve(identity); !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("nested reservation error=%v", err) + } + reservation.release() + reacquired, err := registry.reserve(identity) + if err != nil { + t.Fatalf("reserve identity after release: %v", err) + } + reacquired.release() +} + +func TestProcessRegistryAllowsExactlyOneConcurrentSameIdentityReservation(t *testing.T) { + const contenders = 32 + registry := &processRegistry{} + identity := mustIdentity(t, t.TempDir()) + start := make(chan struct{}) + results := make(chan reservationResult, contenders) + var workers sync.WaitGroup + workers.Add(contenders) + for range contenders { + go func() { + defer workers.Done() + <-start + reservation, err := registry.reserve(identity) + results <- reservationResult{reservation: reservation, err: err} + }() + } + close(start) + workers.Wait() + close(results) + + successes := 0 + nested := 0 + var winner *processReservation + for result := range results { + switch { + case result.err == nil: + successes++ + winner = result.reservation + case errors.Is(result.err, ErrNestedRepositoryAcquisition): + nested++ + default: + t.Fatalf("unexpected reservation error: %v", result.err) + } + } + if successes != 1 || nested != contenders-1 { + t.Fatalf("successes=%d nested=%d want=1/%d", successes, nested, contenders-1) + } + winner.release() +} + +func TestProcessRegistryAllowsConcurrentDifferentIdentities(t *testing.T) { + const repositories = 12 + registry := &processRegistry{} + identities := make([]Identity, repositories) + for i := range identities { + identities[i] = mustIdentity(t, t.TempDir()) + } + + start := make(chan struct{}) + results := make(chan reservationResult, repositories) + var workers sync.WaitGroup + workers.Add(repositories) + for _, identity := range identities { + go func(identity Identity) { + defer workers.Done() + <-start + reservation, err := registry.reserve(identity) + results <- reservationResult{reservation: reservation, err: err} + }(identity) + } + close(start) + workers.Wait() + close(results) + + reservations := make([]*processReservation, 0, repositories) + for result := range results { + if result.err != nil { + t.Fatalf("reserve distinct identity: %v", result.err) + } + reservations = append(reservations, result.reservation) + } + if len(reservations) != repositories { + t.Fatalf("distinct reservations=%d want=%d", len(reservations), repositories) + } + for _, reservation := range reservations { + reservation.release() + } +} + +func TestProcessReservationReleaseIsIdempotentAndCannotRemoveSuccessor(t *testing.T) { + registry := &processRegistry{} + identity := mustIdentity(t, t.TempDir()) + first, err := registry.reserve(identity) + if err != nil { + t.Fatalf("reserve first: %v", err) + } + const releasers = 32 + var workers sync.WaitGroup + workers.Add(releasers) + for range releasers { + go func() { + defer workers.Done() + first.release() + }() + } + workers.Wait() + + second, err := registry.reserve(identity) + if err != nil { + t.Fatalf("reserve successor: %v", err) + } + first.release() + if _, err := registry.reserve(identity); !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("old release removed successor reservation, error=%v", err) + } + second.release() +} + +func TestProcessRegistryCanonicalAliasesCollide(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "repository") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatalf("create repository: %v", err) + } + alias := filepath.Join(root, "alias") + if err := os.Symlink(target, alias); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + direct := mustIdentity(t, target) + throughAlias := mustIdentity(t, alias) + + registry := &processRegistry{} + reservation, err := registry.reserve(direct) + if err != nil { + t.Fatalf("reserve direct identity: %v", err) + } + defer reservation.release() + if _, err := registry.reserve(throughAlias); !errors.Is(err, ErrNestedRepositoryAcquisition) { + t.Fatalf("canonical alias reservation error=%v", err) + } +} + +type reservationResult struct { + reservation *processReservation + err error +} diff --git a/internal/db/migration_backend_contract_test.go b/internal/db/migration_backend_contract_test.go new file mode 100644 index 00000000..558d79d1 --- /dev/null +++ b/internal/db/migration_backend_contract_test.go @@ -0,0 +1,70 @@ +package db_test + +import ( + "testing" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +// SCH-008 is a representative supported SQLite v12 upgrade. The existing +// migrations_test.go fixtures retain focused coverage for v7, v8, v13, and +// other individual historical repair paths. +func TestSCH008SQLiteV12MigrationPreservesStorageBlockData(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{Schema: backendtest.EmptySchema}, func(t *testing.T, backend backendtest.Backend) { + if backend.Kind != db.BackendSQLite { + return + } + mustExec(t, backend.DB, `CREATE TABLE schema_version (version INTEGER PRIMARY KEY)`) + mustExec(t, backend.DB, `INSERT INTO schema_version(version) VALUES (12)`) + mustExec(t, backend.DB, `CREATE TABLE repository_config (key TEXT PRIMARY KEY, value TEXT NOT NULL)`) + mustExec(t, backend.DB, `INSERT INTO repository_config(key, value) VALUES ('default_chunker', 'v1-simple-rolling')`) + mustExec(t, backend.DB, `CREATE TABLE container (id INTEGER PRIMARY KEY, filename TEXT NOT NULL UNIQUE, sealed INTEGER NOT NULL DEFAULT 0, sealing INTEGER NOT NULL DEFAULT 0, quarantine INTEGER NOT NULL DEFAULT 0, current_size INTEGER NOT NULL DEFAULT 0, max_size INTEGER NOT NULL, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)`) + mustExec(t, backend.DB, `CREATE TABLE storage_blocks (id INTEGER PRIMARY KEY, format_version INTEGER NOT NULL, codec TEXT NOT NULL, plaintext_size INTEGER NOT NULL, stored_size INTEGER NOT NULL, container_id INTEGER NOT NULL REFERENCES container(id), container_offset INTEGER NOT NULL, block_hash BLOB NOT NULL, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)`) + mustExec(t, backend.DB, `INSERT INTO container(id, filename, max_size) VALUES ($1,$2,$3)`, 81, "legacy-container", 100) + mustExec(t, backend.DB, `INSERT INTO storage_blocks(id, format_version, codec, plaintext_size, stored_size, container_id, container_offset, block_hash) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, 82, 1, "none", 64, 64, 81, 0, []byte{1, 2, 3}) + if err := db.EnsureSchema(backend.DB); err != nil { + t.Fatalf("migrate SQLite v12 fixture: %v", err) + } + assertCurrentSchemaVersion(t, backend.DB) + var plaintext, stored int64 + var compression string + if err := backend.DB.QueryRow(`SELECT plaintext_size, stored_size, compression_codec FROM storage_blocks WHERE id = $1`, 82).Scan(&plaintext, &stored, &compression); err != nil { + t.Fatalf("read migrated storage block: %v", err) + } + if plaintext != 64 || stored != 64 || compression != "none" { + t.Fatalf("migrated storage block = plaintext:%d stored:%d compression:%q", plaintext, stored, compression) + } + if err := db.EnsureSchema(backend.DB); err != nil { + t.Fatalf("rerun SQLite v12 migration: %v", err) + } + }) +} + +// SCH-009 verifies the production PostgreSQL auto-migration entry point for +// the supported v11 metadata path. The existing PostgreSQL legacy fixtures +// cover v5/v7 snapshots and pre-v6 physical-file migration details. +func TestSCH009PostgresVersionElevenAutoMigration(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + if backend.Kind != db.BackendPostgres { + return + } + mustExec(t, backend.DB, `UPDATE schema_version SET version = 11 WHERE version < 11`) + if err := db.EnsureSchema(backend.DB); err != nil { + t.Fatalf("auto-migrate PostgreSQL v11 fixture: %v", err) + } + assertCurrentSchemaVersion(t, backend.DB) + }) +} + +// SCH-010 and SCH-011 document the current entry points' observable metadata +// behavior. Empty databases bootstrap; an empty version table is rejected by +// CurrentSchemaVersion rather than being misreported as current. +func TestSCH010AndSCH011MetadataBoundaries(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{Schema: backendtest.EmptySchema}, func(t *testing.T, backend backendtest.Backend) { + if err := db.EnsureSchema(backend.DB); err != nil { + t.Fatalf("bootstrap empty database: %v", err) + } + assertCurrentSchemaVersion(t, backend.DB) + }) +} diff --git a/internal/db/mutation_rows.go b/internal/db/mutation_rows.go new file mode 100644 index 00000000..37e1fde8 --- /dev/null +++ b/internal/db/mutation_rows.go @@ -0,0 +1,44 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" +) + +// ErrMutationCardinality identifies a required SQL mutation that did not +// affect the expected number of direct target rows. +var ErrMutationCardinality = errors.New("SQL mutation cardinality mismatch") + +// RequireRowsAffected verifies the direct row count reported by a mutation. +// Operation must be a bounded logical label; callers must not include SQL text, +// connection details, paths, hashes, or bound values. +func RequireRowsAffected(result sql.Result, operation string, expected int64) error { + if result == nil { + return fmt.Errorf("%w: %s: result is nil", ErrMutationCardinality, operation) + } + + actual, err := result.RowsAffected() + if err != nil { + return errors.Join( + ErrMutationCardinality, + fmt.Errorf("%s: determine rows affected: %w", operation, err), + ) + } + if actual != expected { + return fmt.Errorf( + "%w: %s affected %d rows; expected %d", + ErrMutationCardinality, + operation, + actual, + expected, + ) + } + + return nil +} + +// RequireExactlyOneRow verifies that a mutation affected one direct target row. +func RequireExactlyOneRow(result sql.Result, operation string) error { + return RequireRowsAffected(result, operation, 1) +} diff --git a/internal/db/mutation_rows_test.go b/internal/db/mutation_rows_test.go new file mode 100644 index 00000000..db3ed604 --- /dev/null +++ b/internal/db/mutation_rows_test.go @@ -0,0 +1,125 @@ +package db_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +var errRowsAffectedUnsupported = errors.New("rows affected unsupported") + +type unsupportedMutationResult struct{} + +func (unsupportedMutationResult) LastInsertId() (int64, error) { + return 0, errors.New("last insert id unsupported") +} + +func (unsupportedMutationResult) RowsAffected() (int64, error) { + return 0, errRowsAffectedUnsupported +} + +func TestMutationRowsAffectedContractAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{Schema: backendtest.EmptySchema}, func(t *testing.T, backend backendtest.Backend) { + ctx := context.Background() + if _, err := backend.DB.ExecContext(ctx, ` + CREATE TABLE phase17_mutation_rows ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL, + optional_value INTEGER, + flag BOOLEAN NOT NULL DEFAULT FALSE + ) + `); err != nil { + t.Fatalf("create mutation fixture: %v", err) + } + if _, err := backend.DB.ExecContext(ctx, `INSERT INTO phase17_mutation_rows (id, value) VALUES ($1, $2)`, 1, "baseline"); err != nil { + t.Fatalf("insert mutation fixture: %v", err) + } + + existing, err := backend.DB.ExecContext(ctx, `UPDATE phase17_mutation_rows SET value = $1 WHERE id = $2`, "updated", 1) + if err != nil { + t.Fatalf("update existing row: %v", err) + } + if err := db.RequireExactlyOneRow(existing, "update existing fixture"); err != nil { + t.Fatalf("require existing row: %v", err) + } + + sameValue, err := backend.DB.ExecContext(ctx, `UPDATE phase17_mutation_rows SET value = $1 WHERE id = $2`, "updated", 1) + if err != nil { + t.Fatalf("same-value update: %v", err) + } + if err := db.RequireExactlyOneRow(sameValue, "same-value fixture update"); err != nil { + t.Fatalf("require same-value matched row: %v", err) + } + + sameNullableValues, err := backend.DB.ExecContext(ctx, ` + UPDATE phase17_mutation_rows + SET value = $1, optional_value = $2, flag = $3 + WHERE id = $4 + `, "updated", nil, false, 1) + if err != nil { + t.Fatalf("same nullable-value update: %v", err) + } + if err := db.RequireExactlyOneRow(sameNullableValues, "same nullable-value fixture update"); err != nil { + t.Fatalf("require same nullable-value matched row: %v", err) + } + + missing, err := backend.DB.ExecContext(ctx, `UPDATE phase17_mutation_rows SET value = $1 WHERE id = $2`, "missing", 999) + if err != nil { + t.Fatalf("update missing row: %v", err) + } + err = db.RequireExactlyOneRow(missing, "update missing fixture") + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("missing-row error=%v, want ErrMutationCardinality", err) + } + if !strings.Contains(err.Error(), "update missing fixture") || !strings.Contains(err.Error(), "affected 0 rows; expected 1") { + t.Fatalf("missing-row error lacks bounded cardinality details: %v", err) + } + + deleted, err := backend.DB.ExecContext(ctx, `DELETE FROM phase17_mutation_rows WHERE id = $1`, 1) + if err != nil { + t.Fatalf("delete existing row: %v", err) + } + if err := db.RequireExactlyOneRow(deleted, "delete existing fixture"); err != nil { + t.Fatalf("require deleted row: %v", err) + } + + if _, err := backend.DB.ExecContext(ctx, `INSERT INTO phase17_mutation_rows (id, value) VALUES ($1, $2)`, 2, "conflict"); err != nil { + t.Fatalf("insert upsert fixture: %v", err) + } + conflict, err := backend.DB.ExecContext(ctx, ` + INSERT INTO phase17_mutation_rows (id, value) + VALUES ($1, $2) + ON CONFLICT (id) DO NOTHING + `, 2, "ignored") + if err != nil { + t.Fatalf("execute upsert conflict: %v", err) + } + if err := db.RequireRowsAffected(conflict, "upsert conflict fixture", 0); err != nil { + t.Fatalf("require zero-row conflict branch: %v", err) + } + }) + + t.Run("unsupported-result", func(t *testing.T) { + err := db.RequireExactlyOneRow(unsupportedMutationResult{}, "unsupported fixture result") + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if !errors.Is(err, errRowsAffectedUnsupported) { + t.Fatalf("error=%v, want RowsAffected cause", err) + } + if !strings.Contains(err.Error(), "unsupported fixture result") { + t.Fatalf("error lacks operation label: %v", err) + } + }) + + t.Run("nil-result", func(t *testing.T) { + err := db.RequireExactlyOneRow(nil, "nil fixture result") + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) +} diff --git a/internal/db/schema_backend_contract_test.go b/internal/db/schema_backend_contract_test.go new file mode 100644 index 00000000..7be249bf --- /dev/null +++ b/internal/db/schema_backend_contract_test.go @@ -0,0 +1,186 @@ +package db_test + +import ( + "database/sql" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +const phase5SchemaVersion = 16 + +func TestSCH001AndSCH002BootstrapVersionAndIdempotency(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{Schema: backendtest.EmptySchema}, func(t *testing.T, backend backendtest.Backend) { + if err := db.EnsureSchema(backend.DB); err != nil { + t.Fatalf("EnsureSchema: %v", err) + } + assertCurrentSchemaVersion(t, backend.DB) + if _, err := backend.DB.Exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 1, "contract", 1, "hash", 1, "COMPLETED"); err != nil { + t.Fatalf("schema unusable: %v", err) + } + if err := db.EnsureSchema(backend.DB); err != nil { + t.Fatalf("second EnsureSchema: %v", err) + } + assertCurrentSchemaVersion(t, backend.DB) + var name string + if err := backend.DB.QueryRow(`SELECT original_name FROM logical_file WHERE id = $1`, 1).Scan(&name); err != nil { + t.Fatalf("read preserved logical file: %v", err) + } + if name != "contract" { + t.Fatalf("preserved logical-file name = %q, want contract", name) + } + var currentVersionRows int + if err := backend.DB.QueryRow(`SELECT COUNT(*) FROM schema_version WHERE version = $1`, phase5SchemaVersion).Scan(¤tVersionRows); err != nil { + t.Fatalf("count current schema-version rows: %v", err) + } + if currentVersionRows != 1 { + t.Fatalf("current schema-version rows = %d, want 1", currentVersionRows) + } + }) +} + +func TestSCH005PrimaryKeyAndPhysicalFileForeignKey(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + conn := backend.DB + _, err := conn.Exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 1, "one", 1, "hash", 1, "COMPLETED") + if err != nil { + t.Fatal(err) + } + if _, err := conn.Exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 1, "duplicate", 1, "hash2", 1, "COMPLETED"); err == nil { + t.Fatal("duplicate logical file ID succeeded") + } + if _, err := conn.Exec(`INSERT INTO physical_file (path, logical_file_id, mode, mtime, is_metadata_complete) VALUES ($1,$2,$3,$4,$5)`, "/missing", 999, 0o644, time.Now().UTC(), true); err == nil { + t.Fatal("invalid physical-file foreign key succeeded") + } + }) +} + +func TestSCH011CurrentSchemaVersionRejectsEmptyMetadata(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{Schema: backendtest.EmptySchema}, func(t *testing.T, backend backendtest.Backend) { + if _, err := backend.DB.Exec(`CREATE TABLE schema_version (version INTEGER)`); err != nil { + t.Fatal(err) + } + if _, err := db.CurrentSchemaVersion(backend.DB); err == nil { + t.Fatal("empty schema_version accepted") + } + }) +} + +// SCH-003 verifies the current metadata shape is stable and readable without +// requiring physical SQLite/PostgreSQL type equality. +func TestSCH003CurrentSchemaMetadata(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + assertCurrentSchemaVersion(t, backend.DB) + var rows int + if err := backend.DB.QueryRow(`SELECT COUNT(*) FROM schema_version WHERE version = $1`, phase5SchemaVersion).Scan(&rows); err != nil { + t.Fatalf("read current schema metadata: %v", err) + } + if rows != 1 { + t.Fatalf("current schema metadata rows = %d, want 1", rows) + } + }) +} + +// SCH-004 proves only the smallest catalog-shaped write/read operation needed +// after bootstrap; catalog-operation parity belongs to Phase 6. +func TestSCH004MinimalCatalogUsability(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + if _, err := backend.DB.Exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 41, "minimal", 9, "minimal-hash", 1, "COMPLETED"); err != nil { + t.Fatalf("insert minimal logical file: %v", err) + } + var got string + if err := backend.DB.QueryRow(`SELECT file_hash FROM logical_file WHERE id = $1`, 41).Scan(&got); err != nil { + t.Fatalf("read minimal logical file: %v", err) + } + if got != "minimal-hash" { + t.Fatalf("minimal logical-file hash = %q", got) + } + }) +} + +// SCH-005 exercises identity constraints as observable success/failure +// behavior, intentionally not backend-specific error text. +func TestSCH005CriticalUniqueness(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + mustExec(t, backend.DB, `INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 51, "one", 10, "identity-hash", 1, "COMPLETED") + mustFail(t, backend.DB, `INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 52, "two", 10, "identity-hash", 1, "COMPLETED") + mustExec(t, backend.DB, `INSERT INTO snapshot (id, created_at, type) VALUES ($1,$2,$3)`, "snapshot-identity", time.Unix(1, 0).UTC(), "full") + mustFail(t, backend.DB, `INSERT INTO snapshot (id, created_at, type) VALUES ($1,$2,$3)`, "snapshot-identity", time.Unix(2, 0).UTC(), "full") + mustExec(t, backend.DB, `INSERT INTO chunk (id, chunk_hash, size, status) VALUES ($1,$2,$3,$4)`, 53, "chunk-identity", 3, "COMPLETED") + mustFail(t, backend.DB, `INSERT INTO chunk (id, chunk_hash, size, status) VALUES ($1,$2,$3,$4)`, 54, "chunk-identity", 3, "COMPLETED") + mustExec(t, backend.DB, `INSERT INTO physical_file (path, logical_file_id) VALUES ($1,$2)`, "/identity", 51) + mustFail(t, backend.DB, `INSERT INTO physical_file (path, logical_file_id) VALUES ($1,$2)`, "/identity", 51) + }) +} + +// SCH-006 verifies the documented physical-file cascade and restrictive +// dependent references, including SQLite's connection-local foreign-key mode. +func TestSCH006CriticalForeignKeys(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + mustFail(t, backend.DB, `INSERT INTO physical_file (path, logical_file_id) VALUES ($1,$2)`, "/missing", 999) + mustFail(t, backend.DB, `INSERT INTO file_chunk (logical_file_id, chunk_id, chunk_order) VALUES ($1,$2,$3)`, 999, 999, 0) + mustExec(t, backend.DB, `INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 61, "parent", 1, "parent-hash", 1, "COMPLETED") + mustExec(t, backend.DB, `INSERT INTO physical_file (path, logical_file_id) VALUES ($1,$2)`, "/cascaded", 61) + mustExec(t, backend.DB, `DELETE FROM logical_file WHERE id = $1`, 61) + var children int + if err := backend.DB.QueryRow(`SELECT COUNT(*) FROM physical_file WHERE path = $1`, "/cascaded").Scan(&children); err != nil { + t.Fatalf("count cascaded physical file: %v", err) + } + if children != 0 { + t.Fatalf("physical-file cascade left %d row(s)", children) + } + }) +} + +// SCH-007 compares logical defaults and nullable values, not their driver +// storage representation. +func TestSCH007NullableAndDefaultSemantics(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + mustExec(t, backend.DB, `INSERT INTO container (id, filename, max_size) VALUES ($1,$2,$3)`, 71, "defaults", 100) + var sealed, sealing, quarantine bool + var currentSize int64 + if err := backend.DB.QueryRow(`SELECT sealed, sealing, quarantine, current_size FROM container WHERE id = $1`, 71).Scan(&sealed, &sealing, &quarantine, ¤tSize); err != nil { + t.Fatalf("read container defaults: %v", err) + } + if sealed || sealing || quarantine || currentSize != 0 { + t.Fatalf("container defaults = sealed:%t sealing:%t quarantine:%t size:%d", sealed, sealing, quarantine, currentSize) + } + mustExec(t, backend.DB, `INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1,$2,$3,$4,$5,$6)`, 72, "nullable", 1, "nullable-hash", 1, "COMPLETED") + mustExec(t, backend.DB, `INSERT INTO physical_file (path, logical_file_id) VALUES ($1,$2)`, "/nullable", 72) + var mtime sql.NullTime + var complete bool + if err := backend.DB.QueryRow(`SELECT mtime, is_metadata_complete FROM physical_file WHERE path = $1`, "/nullable").Scan(&mtime, &complete); err != nil { + t.Fatalf("read nullable physical metadata: %v", err) + } + if mtime.Valid || complete { + t.Fatalf("physical metadata defaults = mtime valid:%t complete:%t", mtime.Valid, complete) + } + }) +} + +func mustExec(t *testing.T, conn *sql.DB, query string, args ...any) { + t.Helper() + if _, err := conn.Exec(query, args...); err != nil { + t.Fatalf("exec %q: %v", query, err) + } +} + +func mustFail(t *testing.T, conn *sql.DB, query string, args ...any) { + t.Helper() + if _, err := conn.Exec(query, args...); err == nil { + t.Fatalf("expected failure for %q", query) + } +} + +func assertCurrentSchemaVersion(t *testing.T, conn *sql.DB) { + t.Helper() + version, err := db.CurrentSchemaVersion(conn) + if err != nil { + t.Fatalf("CurrentSchemaVersion: %v", err) + } + if version != phase5SchemaVersion { + t.Fatalf("version = %d, want %d", version, phase5SchemaVersion) + } +} diff --git a/internal/db/transaction_backend_contract_helpers_test.go b/internal/db/transaction_backend_contract_helpers_test.go new file mode 100644 index 00000000..9a30a70f --- /dev/null +++ b/internal/db/transaction_backend_contract_helpers_test.go @@ -0,0 +1,276 @@ +package db_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/lib/pq" +) + +const ( + phase10ObserverTimeout = 2 * time.Second + phase10OperationTimeout = 5 * time.Second +) + +type phase10QueryResult struct { + id int64 + value string + err error +} + +type phase10PostgresConnections struct { + a *sql.Conn + b *sql.Conn + observer *sql.Conn +} + +func setupPhase10TransactionTable(t *testing.T, dbconn *sql.DB) { + t.Helper() + phase10Exec(t, dbconn, ` + CREATE TABLE phase10_txn_contract ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL UNIQUE + ) + `) + phase10Exec(t, dbconn, `INSERT INTO phase10_txn_contract (id, value) VALUES ($1, $2)`, 1, "baseline") +} + +func setupPhase10LockTable(t *testing.T, dbconn *sql.DB) { + t.Helper() + phase10Exec(t, dbconn, ` + CREATE TABLE phase10_lock_contract ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ) + `) + phase10Exec(t, dbconn, `INSERT INTO phase10_lock_contract (id, value) VALUES ($1, $2), ($3, $4)`, + 1, "first", 2, "second") +} + +func phase10Exec(t *testing.T, dbconn *sql.DB, query string, args ...any) { + t.Helper() + if _, err := dbconn.Exec(query, args...); err != nil { + t.Fatalf("execute Phase 10 fixture query: %v", err) + } +} + +func openPhase10PostgresConnections(t *testing.T, dbconn *sql.DB) phase10PostgresConnections { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + open := func(role string) *sql.Conn { + t.Helper() + conn, err := dbconn.Conn(ctx) + if err != nil { + t.Fatalf("reserve PostgreSQL %s connection: %v", role, err) + } + t.Cleanup(func() { + if err := conn.Close(); err != nil && !errors.Is(err, sql.ErrConnDone) { + t.Errorf("close PostgreSQL %s connection: %v", role, err) + } + }) + return conn + } + + return phase10PostgresConnections{ + a: open("locker"), + b: open("contender"), + observer: open("observer"), + } +} + +func postgresBackendPID(t *testing.T, conn *sql.Conn) int { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + var pid int + if err := conn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&pid); err != nil { + t.Fatalf("query PostgreSQL backend PID: %v", err) + } + return pid +} + +func waitForPostgresLockWait(t *testing.T, observer *sql.Conn, pid int) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), phase10ObserverTimeout) + defer cancel() + + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for { + var state string + var waitEventType sql.NullString + err := observer.QueryRowContext(ctx, ` + SELECT state, wait_event_type + FROM pg_stat_activity + WHERE pid = $1 + `, pid).Scan(&state, &waitEventType) + if err == nil && state == "active" && waitEventType.Valid && waitEventType.String == "Lock" { + return + } + if err != nil && !errors.Is(err, sql.ErrNoRows) && ctx.Err() == nil { + t.Fatalf("observe PostgreSQL lock wait for pid %d: %v", pid, err) + } + + select { + case <-ctx.Done(): + t.Fatalf("PostgreSQL pid %d did not enter a server-observed lock wait: %v", pid, ctx.Err()) + case <-ticker.C: + } + } +} + +func startPhase10BlockingQuery( + ctx context.Context, + tx *sql.Tx, + query string, + args ...any, +) (<-chan phase10QueryResult, *sync.WaitGroup) { + results := make(chan phase10QueryResult, 1) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + var id int64 + var value string + err := tx.QueryRowContext(ctx, query, args...).Scan(&id, &value) + results <- phase10QueryResult{id: id, value: value, err: err} + }() + return results, &wg +} + +func awaitPhase10QueryResult(t *testing.T, results <-chan phase10QueryResult, wg *sync.WaitGroup) phase10QueryResult { + t.Helper() + timer := time.NewTimer(phase10OperationTimeout) + defer timer.Stop() + + select { + case result := <-results: + wg.Wait() + return result + case <-timer.C: + t.Fatal("timed out waiting for Phase 10 blocked query to finish") + return phase10QueryResult{} + } +} + +func assertPhase10PostgresCode(t *testing.T, err error, code string) { + t.Helper() + var pqErr *pq.Error + if !errors.As(err, &pqErr) { + t.Fatalf("expected PostgreSQL SQLSTATE %s, got %T: %v", code, err, err) + } + if got := string(pqErr.Code); got != code { + t.Fatalf("expected PostgreSQL SQLSTATE %s, got %s: %v", code, got, err) + } +} + +func assertPhase10Cancellation(t *testing.T, ctx context.Context, err error) { + t.Helper() + if err == nil { + t.Fatal("expected blocked PostgreSQL query cancellation error") + } + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatalf("expected cancelled query context, got %v", ctx.Err()) + } + if errors.Is(err, context.Canceled) { + return + } + + var pqErr *pq.Error + if errors.As(err, &pqErr) && string(pqErr.Code) == "57014" { + return + } + t.Fatalf("expected context cancellation or PostgreSQL SQLSTATE 57014, got %T: %v", err, err) +} + +func assertPhase10ConnectionReusable(t *testing.T, dbconn *sql.DB) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + var one int + if err := dbconn.QueryRowContext(ctx, `SELECT 1`).Scan(&one); err != nil { + t.Fatalf("final connection reuse query: %v", err) + } + if one != 1 { + t.Fatalf("final connection reuse query returned %d", one) + } +} + +func assertPhase10LockRowsUnchanged(t *testing.T, dbconn *sql.DB) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + rows, err := dbconn.QueryContext(ctx, ` + SELECT id, value + FROM phase10_lock_contract + ORDER BY id + `) + if err != nil { + t.Fatalf("query final Phase 10 lock rows: %v", err) + } + defer func() { + if err := rows.Close(); err != nil { + t.Errorf("close final Phase 10 lock rows: %v", err) + } + }() + + want := []phase10QueryResult{ + {id: 1, value: "first"}, + {id: 2, value: "second"}, + } + for index, expected := range want { + if !rows.Next() { + t.Fatalf("missing final Phase 10 lock row %d", expected.id) + } + var id int64 + var value string + if err := rows.Scan(&id, &value); err != nil { + t.Fatalf("scan final Phase 10 lock row %d: %v", expected.id, err) + } + if id != expected.id || value != expected.value { + t.Fatalf( + "final Phase 10 lock row %d = (%d, %q), want (%d, %q)", + index, + id, + value, + expected.id, + expected.value, + ) + } + } + if rows.Next() { + t.Fatal("unexpected extra final Phase 10 lock row") + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate final Phase 10 lock rows: %v", err) + } +} + +func rollbackPhase10Tx(t *testing.T, tx **sql.Tx, role string) { + t.Helper() + if tx == nil || *tx == nil { + return + } + if err := (*tx).Rollback(); err != nil && + !errors.Is(err, sql.ErrTxDone) && + !errors.Is(err, driver.ErrBadConn) { + t.Errorf("rollback Phase 10 %s transaction: %v", role, err) + } + *tx = nil +} + +func phase10LockQuery(base string, clause string) string { + return fmt.Sprintf("%s %s", base, clause) +} diff --git a/internal/db/transaction_backend_contract_test.go b/internal/db/transaction_backend_contract_test.go new file mode 100644 index 00000000..4497ecee --- /dev/null +++ b/internal/db/transaction_backend_contract_test.go @@ -0,0 +1,564 @@ +package db_test + +import ( + "context" + "errors" + "testing" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestBackendTransactionCommitRollbackAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + setupPhase10TransactionTable(t, backend.DB) + + if got := db.BackendFromDB(backend.DB); got != backend.Kind { + t.Fatalf("backend detection mismatch: got %q want %q", got, backend.Kind) + } + wantLocks := backend.Kind == db.BackendPostgres + if backend.Capabilities.SelectForUpdate != wantLocks || + backend.Capabilities.Nowait != wantLocks || + backend.Capabilities.SkipLocked != wantLocks { + t.Fatalf("unexpected lock capabilities for %s: %+v", backend.Name, backend.Capabilities) + } + const capabilityQuery = `SELECT id FROM phase10_txn_contract WHERE id = $1` + wantForUpdate := capabilityQuery + wantNowait := capabilityQuery + wantSkipLocked := capabilityQuery + if wantLocks { + wantForUpdate = phase10LockQuery(capabilityQuery, "FOR UPDATE") + wantNowait = phase10LockQuery(capabilityQuery, "FOR UPDATE NOWAIT") + wantSkipLocked = phase10LockQuery(capabilityQuery, "FOR UPDATE SKIP LOCKED") + } + if got := db.QueryWithOptionalForUpdate(backend.DB, capabilityQuery); got != wantForUpdate { + t.Fatalf("%s FOR UPDATE query=%q want %q", backend.Name, got, wantForUpdate) + } + if got := db.QueryWithOptionalForUpdateNowait(backend.DB, capabilityQuery); got != wantNowait { + t.Fatalf("%s FOR UPDATE NOWAIT query=%q want %q", backend.Name, got, wantNowait) + } + if got := db.QueryWithOptionalForUpdateSkipLocked(backend.DB, capabilityQuery); got != wantSkipLocked { + t.Fatalf("%s FOR UPDATE SKIP LOCKED query=%q want %q", backend.Name, got, wantSkipLocked) + } + + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + commitTx, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin commit transaction: %v", err) + } + if _, err := commitTx.ExecContext(ctx, + `INSERT INTO phase10_txn_contract (id, value) VALUES ($1, $2)`, + 2, "committed", + ); err != nil { + _ = commitTx.Rollback() + t.Fatalf("insert committed fixture row: %v", err) + } + var ownWrite string + if err := commitTx.QueryRowContext(ctx, + `SELECT value FROM phase10_txn_contract WHERE id = $1`, 2, + ).Scan(&ownWrite); err != nil { + _ = commitTx.Rollback() + t.Fatalf("read own insert: %v", err) + } + if ownWrite != "committed" { + _ = commitTx.Rollback() + t.Fatalf("read own insert value=%q", ownWrite) + } + if err := commitTx.Commit(); err != nil { + t.Fatalf("commit insertion: %v", err) + } + + var committed string + if err := backend.DB.QueryRowContext(ctx, + `SELECT value FROM phase10_txn_contract WHERE id = $1`, 2, + ).Scan(&committed); err != nil { + t.Fatalf("read committed insertion: %v", err) + } + if committed != "committed" { + t.Fatalf("committed value=%q", committed) + } + + rollbackTx, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin rollback transaction: %v", err) + } + if _, err := rollbackTx.ExecContext(ctx, + `UPDATE phase10_txn_contract SET value = $1 WHERE id = $2`, + "rolled-back", 1, + ); err != nil { + _ = rollbackTx.Rollback() + t.Fatalf("update rollback fixture row: %v", err) + } + if err := rollbackTx.QueryRowContext(ctx, + `SELECT value FROM phase10_txn_contract WHERE id = $1`, 1, + ).Scan(&ownWrite); err != nil { + _ = rollbackTx.Rollback() + t.Fatalf("read own update: %v", err) + } + if ownWrite != "rolled-back" { + _ = rollbackTx.Rollback() + t.Fatalf("read own update value=%q", ownWrite) + } + if err := rollbackTx.Rollback(); err != nil { + t.Fatalf("rollback update: %v", err) + } + + var baseline string + if err := backend.DB.QueryRowContext(ctx, + `SELECT value FROM phase10_txn_contract WHERE id = $1`, 1, + ).Scan(&baseline); err != nil { + t.Fatalf("read row after rollback: %v", err) + } + if baseline != "baseline" { + t.Fatalf("rollback did not restore baseline: %q", baseline) + } + + conflictTx, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin constraint transaction: %v", err) + } + if _, err := conflictTx.ExecContext(ctx, + `INSERT INTO phase10_txn_contract (id, value) VALUES ($1, $2)`, + 3, "temporary", + ); err != nil { + _ = conflictTx.Rollback() + t.Fatalf("insert pre-conflict row: %v", err) + } + if _, err := conflictTx.ExecContext(ctx, + `INSERT INTO phase10_txn_contract (id, value) VALUES ($1, $2)`, + 4, "baseline", + ); err == nil { + _ = conflictTx.Rollback() + t.Fatal("expected uniqueness conflict") + } + if err := conflictTx.Rollback(); err != nil { + t.Fatalf("rollback uniqueness conflict: %v", err) + } + + var temporaryCount int + if err := backend.DB.QueryRowContext(ctx, + `SELECT COUNT(*) FROM phase10_txn_contract WHERE id = $1`, 3, + ).Scan(&temporaryCount); err != nil { + t.Fatalf("count pre-conflict row after rollback: %v", err) + } + if temporaryCount != 0 { + t.Fatalf("constraint rollback retained %d pre-conflict rows", temporaryCount) + } + + affectedTx, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin affected-row transaction: %v", err) + } + existingResult, err := affectedTx.ExecContext(ctx, + `UPDATE phase10_txn_contract SET value = $1 WHERE id = $2`, + "affected", 1, + ) + if err != nil { + _ = affectedTx.Rollback() + t.Fatalf("update existing affected-row fixture: %v", err) + } + existingRows, err := existingResult.RowsAffected() + if err != nil { + _ = affectedTx.Rollback() + t.Fatalf("existing RowsAffected: %v", err) + } + if existingRows != 1 { + _ = affectedTx.Rollback() + t.Fatalf("existing RowsAffected=%d want 1", existingRows) + } + missingResult, err := affectedTx.ExecContext(ctx, + `UPDATE phase10_txn_contract SET value = $1 WHERE id = $2`, + "missing", 999, + ) + if err != nil { + _ = affectedTx.Rollback() + t.Fatalf("update missing affected-row fixture: %v", err) + } + missingRows, err := missingResult.RowsAffected() + if err != nil { + _ = affectedTx.Rollback() + t.Fatalf("missing RowsAffected: %v", err) + } + if missingRows != 0 { + _ = affectedTx.Rollback() + t.Fatalf("missing RowsAffected=%d want 0", missingRows) + } + if err := affectedTx.Rollback(); err != nil { + t.Fatalf("rollback affected-row transaction: %v", err) + } + + cancelled, cancelNow := context.WithCancel(context.Background()) + cancelNow() + cancelledTx, err := backend.DB.BeginTx(cancelled, nil) + if cancelledTx != nil { + _ = cancelledTx.Rollback() + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("pre-cancelled BeginTx error=%v", err) + } + if _, err := backend.DB.ExecContext(cancelled, + `UPDATE phase10_txn_contract SET value = $1 WHERE id = $2`, + "cancelled", 1, + ); !errors.Is(err, context.Canceled) { + t.Fatalf("pre-cancelled update error=%v", err) + } + + if err := backend.DB.QueryRowContext(ctx, + `SELECT value FROM phase10_txn_contract WHERE id = $1`, 1, + ).Scan(&baseline); err != nil { + t.Fatalf("read final baseline: %v", err) + } + if baseline != "baseline" { + t.Fatalf("final baseline=%q", baseline) + } + assertPhase10ConnectionReusable(t, backend.DB) + }) +} + +func TestBackendForUpdateLockReleaseAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + setupPhase10LockTable(t, backend.DB) + const baseQuery = `SELECT id, value FROM phase10_lock_contract WHERE id = $1` + lockQuery := db.QueryWithOptionalForUpdate(backend.DB, baseQuery) + + if backend.Kind == db.BackendSQLite { + if lockQuery != baseQuery { + t.Fatalf("SQLite FOR UPDATE helper emitted %q", lockQuery) + } + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + tx, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin SQLite boundary transaction: %v", err) + } + var id int64 + var value string + if err := tx.QueryRowContext(ctx, lockQuery, 1).Scan(&id, &value); err != nil { + _ = tx.Rollback() + t.Fatalf("execute SQLite lock-clause boundary query: %v", err) + } + if id != 1 || value != "first" { + _ = tx.Rollback() + t.Fatalf("SQLite boundary selected (%d, %q), want (1, %q)", id, value, "first") + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback SQLite boundary transaction: %v", err) + } + assertPhase10LockRowsUnchanged(t, backend.DB) + assertPhase10ConnectionReusable(t, backend.DB) + return + } + if lockQuery != phase10LockQuery(baseQuery, "FOR UPDATE") { + t.Fatalf("PostgreSQL FOR UPDATE helper emitted %q", lockQuery) + } + + for _, release := range []string{"commit", "rollback"} { + t.Run(release+"_release", func(t *testing.T) { + conns := openPhase10PostgresConnections(t, backend.DB) + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + txA, err := conns.a.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin locker transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txA, "locker") + var lockedID int64 + var lockedValue string + if err := txA.QueryRowContext(ctx, lockQuery, 1).Scan(&lockedID, &lockedValue); err != nil { + t.Fatalf("acquire PostgreSQL row lock: %v", err) + } + if lockedID != 1 || lockedValue != "first" { + t.Fatalf("locker selected (%d, %q), want (1, %q)", lockedID, lockedValue, "first") + } + + pidB := postgresBackendPID(t, conns.b) + txB, err := conns.b.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin contender transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txB, "contender") + results, wg := startPhase10BlockingQuery(ctx, txB, lockQuery, 1) + waitForPostgresLockWait(t, conns.observer, pidB) + + if release == "commit" { + if err := txA.Commit(); err != nil { + t.Fatalf("commit locker transaction: %v", err) + } + } else { + if err := txA.Rollback(); err != nil { + t.Fatalf("rollback locker transaction: %v", err) + } + } + txA = nil + + result := awaitPhase10QueryResult(t, results, wg) + if result.err != nil { + t.Fatalf("contender acquire after %s: %v", release, result.err) + } + if result.id != 1 || result.value != "first" { + t.Fatalf( + "contender selected (%d, %q) after %s, want (1, %q)", + result.id, + result.value, + release, + "first", + ) + } + if err := txB.Commit(); err != nil { + t.Fatalf("commit contender after %s: %v", release, err) + } + txB = nil + }) + } + + assertPhase10LockRowsUnchanged(t, backend.DB) + assertPhase10ConnectionReusable(t, backend.DB) + }) +} + +func TestBackendNowaitAndSkipLockedAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + setupPhase10LockTable(t, backend.DB) + const rowQuery = `SELECT id FROM phase10_lock_contract WHERE id = $1` + const candidateQuery = `SELECT id FROM phase10_lock_contract ORDER BY id ASC LIMIT 1` + nowaitQuery := db.QueryWithOptionalForUpdateNowait(backend.DB, rowQuery) + skipLockedQuery := db.QueryWithOptionalForUpdateSkipLocked(backend.DB, candidateQuery) + + if backend.Kind == db.BackendSQLite { + if nowaitQuery != rowQuery || skipLockedQuery != candidateQuery { + t.Fatalf("SQLite emitted PostgreSQL lock clauses: nowait=%q skip=%q", nowaitQuery, skipLockedQuery) + } + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + tx, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin SQLite lock boundary transaction: %v", err) + } + var id int64 + if err := tx.QueryRowContext(ctx, skipLockedQuery).Scan(&id); err != nil { + _ = tx.Rollback() + t.Fatalf("execute SQLite ordered candidate query: %v", err) + } + if id != 1 { + _ = tx.Rollback() + t.Fatalf("SQLite ordered candidate id=%d want 1", id) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback SQLite lock boundary transaction: %v", err) + } + assertPhase10ConnectionReusable(t, backend.DB) + return + } + + t.Run("nowait", func(t *testing.T) { + conns := openPhase10PostgresConnections(t, backend.DB) + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + txA, err := conns.a.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin NOWAIT locker transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txA, "NOWAIT locker") + var id int64 + if err := txA.QueryRowContext(ctx, db.QueryWithOptionalForUpdate(backend.DB, rowQuery), 1).Scan(&id); err != nil { + t.Fatalf("acquire NOWAIT fixture lock: %v", err) + } + if id != 1 { + t.Fatalf("NOWAIT locker selected id=%d want 1", id) + } + + txB, err := conns.b.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin NOWAIT contender transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txB, "NOWAIT contender") + nowaitCtx, nowaitCancel := context.WithTimeout(context.Background(), phase10ObserverTimeout) + defer nowaitCancel() + err = txB.QueryRowContext(nowaitCtx, nowaitQuery, 1).Scan(&id) + if err == nil { + t.Fatal("expected NOWAIT lock conflict") + } + assertPhase10PostgresCode(t, err, "55P03") + if err := txB.Rollback(); err != nil { + t.Fatalf("rollback NOWAIT contender: %v", err) + } + txB = nil + + if err := txA.QueryRowContext(ctx, `SELECT id FROM phase10_lock_contract WHERE id = $1`, 1).Scan(&id); err != nil { + t.Fatalf("locker transaction invalid after contender NOWAIT failure: %v", err) + } + if id != 1 { + t.Fatalf("locker transaction reuse selected id=%d want 1", id) + } + if err := txA.Rollback(); err != nil { + t.Fatalf("release NOWAIT locker: %v", err) + } + txA = nil + + txC, err := conns.b.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin post-release NOWAIT transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txC, "post-release NOWAIT") + if err := txC.QueryRowContext(ctx, nowaitQuery, 1).Scan(&id); err != nil { + t.Fatalf("NOWAIT acquisition after release: %v", err) + } + if id != 1 { + t.Fatalf("post-release NOWAIT selected id=%d", id) + } + if err := txC.Rollback(); err != nil { + t.Fatalf("rollback post-release NOWAIT transaction: %v", err) + } + txC = nil + }) + + t.Run("skip_locked", func(t *testing.T) { + conns := openPhase10PostgresConnections(t, backend.DB) + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + txA, err := conns.a.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin SKIP LOCKED locker transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txA, "SKIP LOCKED locker") + var id int64 + if err := txA.QueryRowContext(ctx, db.QueryWithOptionalForUpdate(backend.DB, rowQuery), 1).Scan(&id); err != nil { + t.Fatalf("acquire SKIP LOCKED fixture lock: %v", err) + } + if id != 1 { + t.Fatalf("SKIP LOCKED locker selected id=%d want 1", id) + } + + txB, err := conns.b.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin SKIP LOCKED contender transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txB, "SKIP LOCKED contender") + if err := txB.QueryRowContext(ctx, skipLockedQuery).Scan(&id); err != nil { + t.Fatalf("select unlocked candidate: %v", err) + } + if id != 2 { + t.Fatalf("SKIP LOCKED selected id=%d want 2", id) + } + if err := txB.Commit(); err != nil { + t.Fatalf("commit SKIP LOCKED contender: %v", err) + } + txB = nil + + if err := txA.Rollback(); err != nil { + t.Fatalf("release SKIP LOCKED locker: %v", err) + } + txA = nil + + txC, err := conns.b.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin post-release SKIP LOCKED transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txC, "post-release SKIP LOCKED") + if err := txC.QueryRowContext(ctx, skipLockedQuery).Scan(&id); err != nil { + t.Fatalf("select candidate after lock release: %v", err) + } + if id != 1 { + t.Fatalf("post-release candidate id=%d want 1", id) + } + if err := txC.Rollback(); err != nil { + t.Fatalf("rollback post-release SKIP LOCKED transaction: %v", err) + } + txC = nil + }) + + assertPhase10LockRowsUnchanged(t, backend.DB) + assertPhase10ConnectionReusable(t, backend.DB) + }) +} + +func TestBackendBlockedLockCancellationAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + setupPhase10LockTable(t, backend.DB) + const baseQuery = `SELECT id, value FROM phase10_lock_contract WHERE id = $1` + lockQuery := db.QueryWithOptionalForUpdate(backend.DB, baseQuery) + + if backend.Kind == db.BackendSQLite { + if lockQuery != baseQuery { + t.Fatalf("SQLite blocked-lock boundary emitted %q", lockQuery) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + var id int64 + var value string + if err := backend.DB.QueryRowContext(cancelled, lockQuery, 1).Scan(&id, &value); !errors.Is(err, context.Canceled) { + t.Fatalf("SQLite cancelled lock-boundary query error=%v", err) + } + assertPhase10LockRowsUnchanged(t, backend.DB) + assertPhase10ConnectionReusable(t, backend.DB) + return + } + + conns := openPhase10PostgresConnections(t, backend.DB) + ctx, cancel := context.WithTimeout(context.Background(), phase10OperationTimeout) + defer cancel() + + txA, err := conns.a.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin cancellation locker transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txA, "cancellation locker") + var id int64 + var value string + if err := txA.QueryRowContext(ctx, lockQuery, 1).Scan(&id, &value); err != nil { + t.Fatalf("acquire cancellation fixture lock: %v", err) + } + if id != 1 || value != "first" { + t.Fatalf("cancellation locker selected (%d, %q), want (1, %q)", id, value, "first") + } + + pidB := postgresBackendPID(t, conns.b) + txB, err := conns.b.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin cancellation contender transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txB, "cancellation contender") + + queryCtx, queryCancel := context.WithCancel(context.Background()) + results, wg := startPhase10BlockingQuery(queryCtx, txB, lockQuery, 1) + waitForPostgresLockWait(t, conns.observer, pidB) + queryCancel() + + result := awaitPhase10QueryResult(t, results, wg) + assertPhase10Cancellation(t, queryCtx, result.err) + // A cancelled lib/pq query can make database/sql discard B's physical + // connection. rollbackPhase10Tx accepts that terminal cleanup outcome + // while still reporting any other rollback failure. + rollbackPhase10Tx(t, &txB, "cancelled contender") + if err := txA.Rollback(); err != nil { + t.Fatalf("release cancellation locker: %v", err) + } + txA = nil + + // Cancellation can retire B's dedicated connection, so the later + // acquisition must come from the pool rather than that closed handle. + txAfter, err := backend.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin post-cancellation transaction: %v", err) + } + defer rollbackPhase10Tx(t, &txAfter, "post-cancellation") + if err := txAfter.QueryRowContext(ctx, lockQuery, 1).Scan(&id, &value); err != nil { + t.Fatalf("acquire row after cancellation cleanup: %v", err) + } + if id != 1 || value != "first" { + t.Fatalf("post-cancellation query selected (%d, %q), want (1, %q)", id, value, "first") + } + if err := txAfter.Rollback(); err != nil { + t.Fatalf("rollback post-cancellation transaction: %v", err) + } + txAfter = nil + + assertPhase10LockRowsUnchanged(t, backend.DB) + assertPhase10ConnectionReusable(t, backend.DB) + }) +} diff --git a/internal/engine/contract_surface_clarity_test.go b/internal/engine/contract_surface_clarity_test.go index 0b26d039..9c5cd8c0 100644 --- a/internal/engine/contract_surface_clarity_test.go +++ b/internal/engine/contract_surface_clarity_test.go @@ -3,15 +3,16 @@ package engine import "testing" func TestSnapshotQueryOrNilReturnsNilForEmptyQuery(t *testing.T) { - if got := snapshotQueryOrNil(SnapshotQuery{}); got != nil { - t.Fatalf("snapshotQueryOrNil(empty): got %#v, want nil", got) + got, err := snapshotQueryOrNil(SnapshotQuery{}) + if err != nil || got != nil { + t.Fatalf("snapshotQueryOrNil(empty): got (%#v, %v), want (nil, nil)", got, err) } } func TestEngineQueryToSnapshotQueryPreservesSinglePathAndPrefixShape(t *testing.T) { min := int64(1) max := int64(8) - got := engineQueryToSnapshotQuery(SnapshotQuery{ + got, err := engineQueryToSnapshotQuery(SnapshotQuery{ Path: "docs/a.txt", Prefix: "docs/", Pattern: "*.txt", @@ -20,6 +21,9 @@ func TestEngineQueryToSnapshotQueryPreservesSinglePathAndPrefixShape(t *testing. MaxSize: &max, Limit: 5, }) + if err != nil { + t.Fatalf("engineQueryToSnapshotQuery: %v", err) + } if got == nil { t.Fatal("engineQueryToSnapshotQuery: got nil") } @@ -34,6 +38,12 @@ func TestEngineQueryToSnapshotQueryPreservesSinglePathAndPrefixShape(t *testing. } } +func TestEngineQueryToSnapshotQueryRejectsInvalidRegex(t *testing.T) { + if _, err := engineQueryToSnapshotQuery(SnapshotQuery{Regex: "("}); err == nil { + t.Fatal("expected invalid regex error") + } +} + func TestBuildSnapshotDiffResultSummaryModeOmitsEntries(t *testing.T) { result := buildSnapshotDiffResult( SnapshotDiffRequest{BaseID: "base", TargetID: "target", Summary: true}, diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index 368f6c84..bcf6fc46 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -98,6 +98,9 @@ func (e *DefaultEngine) Inspect(ctx context.Context, req InspectRequest) (Inspec } func (e *DefaultEngine) Verify(ctx context.Context, req VerifyRequest) (VerifyResult, error) { + if err := ctx.Err(); err != nil { + return VerifyResult{}, err + } level, err := verifyLevelFromString(req.Level) if err != nil { return VerifyResult{}, err @@ -229,7 +232,11 @@ func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowReques } var snapshotQ *snapshot.SnapshotQuery if req.Query != (SnapshotQuery{}) { - snapshotQ = engineQueryToSnapshotQuery(req.Query) + var err error + snapshotQ, err = engineQueryToSnapshotQuery(req.Query) + if err != nil { + return SnapshotShowResult{}, err + } } entries, err := snapshot.ListSnapshotFiles(ctx, e.config.DB, req.SnapshotID, req.Query.Limit, snapshotQ) if err != nil { @@ -325,7 +332,7 @@ func (e *DefaultEngine) Restore(ctx context.Context, req RestoreRequest) (Restor // engineQueryToSnapshotQuery maps an engine-level SnapshotQuery to the // snapshot package's equivalent type. -func engineQueryToSnapshotQuery(q SnapshotQuery) *snapshot.SnapshotQuery { +func engineQueryToSnapshotQuery(q SnapshotQuery) (*snapshot.SnapshotQuery, error) { sq := &snapshot.SnapshotQuery{ Pattern: q.Pattern, MinSize: q.MinSize, @@ -340,9 +347,11 @@ func engineQueryToSnapshotQuery(q SnapshotQuery) *snapshot.SnapshotQuery { sq.Prefixes = []string{q.Prefix} } if q.Regex != "" { - if compiled, err := regexp.Compile(q.Regex); err == nil { - sq.Regex = compiled + compiled, err := regexp.Compile(q.Regex) + if err != nil { + return nil, fmt.Errorf("invalid snapshot query regex %q: %w", q.Regex, err) } + sq.Regex = compiled } - return sq + return sq, nil } diff --git a/internal/engine/default_engine_routing_helpers.go b/internal/engine/default_engine_routing_helpers.go index 36d84332..2071be70 100644 --- a/internal/engine/default_engine_routing_helpers.go +++ b/internal/engine/default_engine_routing_helpers.go @@ -42,7 +42,11 @@ func (e *DefaultEngine) snapshotDiffSummaryFastPath(ctx context.Context, req Sna } func (e *DefaultEngine) snapshotDiffDetailed(ctx context.Context, req SnapshotDiffRequest) (SnapshotDiffResult, error) { - raw, err := snapshot.DiffSnapshots(ctx, e.config.DB, req.BaseID, req.TargetID, snapshotQueryOrNil(req.Query)) + query, err := snapshotQueryOrNil(req.Query) + if err != nil { + return SnapshotDiffResult{}, err + } + raw, err := snapshot.DiffSnapshots(ctx, e.config.DB, req.BaseID, req.TargetID, query) if err != nil { return SnapshotDiffResult{}, err } @@ -59,9 +63,9 @@ func (e *DefaultEngine) snapshotDiffDetailed(ctx context.Context, req SnapshotDi return buildSnapshotDiffResult(req, entries, summary, len(raw.Entries)), nil } -func snapshotQueryOrNil(q SnapshotQuery) *snapshot.SnapshotQuery { +func snapshotQueryOrNil(q SnapshotQuery) (*snapshot.SnapshotQuery, error) { if q == (SnapshotQuery{}) { - return nil + return nil, nil } return engineQueryToSnapshotQuery(q) } diff --git a/internal/engine/mutation_backend_contract_helpers_test.go b/internal/engine/mutation_backend_contract_helpers_test.go new file mode 100644 index 00000000..79df4556 --- /dev/null +++ b/internal/engine/mutation_backend_contract_helpers_test.go @@ -0,0 +1,493 @@ +package engine_test + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +var mutationBackendCWDMu sync.Mutex + +type mutationBackendFixture struct { + backend backendtest.Backend + engine *engine.DefaultEngine + writer *container.LocalWriter + storeContext storage.StorageContext + inputRoot string + containerDir string + finalized bool +} + +func newMutationBackendFixture(t *testing.T, backend backendtest.Backend) *mutationBackendFixture { + t.Helper() + inputRoot := filepath.Join(t.TempDir(), "input") + containerDir := filepath.Join(t.TempDir(), "containers") + if err := os.MkdirAll(inputRoot, 0o700); err != nil { + t.Fatalf("create mutation input root: %v", err) + } + writer := container.NewLocalWriterWithDirAndDB( + containerDir, + container.GetContainerMaxSize(), + backend.DB, + ) + storeContext := storage.StorageContext{ + DB: backend.DB, + Writer: writer, + ContainerDir: containerDir, + } + eng, err := engine.New(engine.Config{ + DB: backend.DB, + ContainerDir: containerDir, + StoreContext: &storeContext, + }) + if err != nil { + t.Fatalf("engine.New mutation fixture: %v", err) + } + fixture := &mutationBackendFixture{ + backend: backend, engine: eng, writer: writer, storeContext: storeContext, + inputRoot: inputRoot, containerDir: containerDir, + } + t.Cleanup(func() { + if !fixture.finalized { + _ = fixture.writer.FinalizeContainer() + } + }) + return fixture +} + +func (f *mutationBackendFixture) store(t *testing.T, storedPath string, payload []byte) engine.StoreResult { + t.Helper() + cleanPath := filepath.FromSlash(storedPath) + inputPath := filepath.Join(f.inputRoot, cleanPath) + if err := os.MkdirAll(filepath.Dir(inputPath), 0o700); err != nil { + t.Fatalf("create input parent for %q: %v", storedPath, err) + } + if err := os.WriteFile(inputPath, payload, 0o600); err != nil { + t.Fatalf("write input %q: %v", storedPath, err) + } + return f.storeExisting(t, storedPath) +} + +func (f *mutationBackendFixture) storeExisting(t *testing.T, storedPath string) engine.StoreResult { + t.Helper() + sourcePath := filepath.Join(f.inputRoot, filepath.FromSlash(storedPath)) + absoluteSource, err := filepath.Abs(sourcePath) + if err != nil { + t.Fatalf("resolve Store source %q: %v", storedPath, err) + } + if _, err := f.backend.DB.ExecContext(context.Background(), ` + UPDATE physical_file + SET path = $1 + WHERE path = $2`, absoluteSource, filepath.ToSlash(storedPath)); err != nil { + t.Fatalf("prepare stable replacement path %q: %v", storedPath, err) + } + var result engine.StoreResult + mutationBackendWithCWD(t, f.inputRoot, func() { + var err error + result, err = f.engine.Store(context.Background(), engine.StoreRequest{ + SourcePath: filepath.ToSlash(storedPath), + Codec: "plain", + }) + if err != nil { + t.Fatalf("Store %q: %v", storedPath, err) + } + }) + if _, err := f.backend.DB.ExecContext(context.Background(), ` + UPDATE physical_file + SET path = $1 + WHERE path = $2`, filepath.ToSlash(storedPath), result.StoredPath); err != nil { + t.Fatalf("normalize fixture stored path %q: %v", storedPath, err) + } + return result +} + +func (f *mutationBackendFixture) finalize(t *testing.T) { + t.Helper() + if f.finalized { + return + } + if err := f.writer.FinalizeContainer(); err != nil { + t.Fatalf("finalize mutation fixture container: %v", err) + } + f.finalized = true +} + +func (f *mutationBackendFixture) restartWriter(t *testing.T) { + t.Helper() + if !f.finalized { + t.Fatal("restart mutation writer before finalizing previous container") + } + writer := container.NewLocalWriterWithDirAndDB( + f.containerDir, + container.GetContainerMaxSize(), + f.backend.DB, + ) + f.writer = writer + f.storeContext = storage.StorageContext{ + DB: f.backend.DB, + Writer: writer, + ContainerDir: f.containerDir, + } + eng, err := engine.New(engine.Config{ + DB: f.backend.DB, + ContainerDir: f.containerDir, + StoreContext: &f.storeContext, + }) + if err != nil { + t.Fatalf("engine.New restarted mutation fixture: %v", err) + } + f.engine = eng + f.finalized = false +} + +func (f *mutationBackendFixture) readEngine(t *testing.T) *engine.DefaultEngine { + t.Helper() + storeContext := storage.StorageContext{ + DB: f.backend.DB, + ContainerDir: f.containerDir, + } + eng, err := engine.New(engine.Config{ + DB: f.backend.DB, + ContainerDir: f.containerDir, + StoreContext: &storeContext, + }) + if err != nil { + t.Fatalf("engine.New finalized mutation fixture: %v", err) + } + return eng +} + +func (f *mutationBackendFixture) useAbsoluteStoredPath(t *testing.T, storedPath string) string { + t.Helper() + absolutePath := filepath.Join(f.inputRoot, filepath.FromSlash(storedPath)) + if _, err := f.backend.DB.ExecContext(context.Background(), ` + UPDATE physical_file + SET path = $1 + WHERE path = $2`, absolutePath, filepath.ToSlash(storedPath)); err != nil { + t.Fatalf("set absolute fixture stored path %q: %v", storedPath, err) + } + return absolutePath +} + +func mutationBackendWithCWD(t *testing.T, dir string, fn func()) { + t.Helper() + mutationBackendCWDMu.Lock() + defer mutationBackendCWDMu.Unlock() + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + if err := os.Chdir(dir); err != nil { + t.Fatalf("change working directory to %q: %v", dir, err) + } + defer func() { + if err := os.Chdir(cwd); err != nil { + t.Fatalf("restore working directory to %q: %v", cwd, err) + } + }() + fn() +} + +type mutationRepositoryFingerprint struct { + tableCounts map[string]int64 + logical []string + physical []string + fileChunks []string + chunks []string + legacy []string + packed []string + packedRefs []string + containers []string + snapshots []string + snapshotPaths []string + members []string + files []string +} + +func captureMutationRepositoryFingerprint( + t *testing.T, + dbconn *sql.DB, + containerDir string, +) mutationRepositoryFingerprint { + t.Helper() + fingerprint := mutationRepositoryFingerprint{tableCounts: make(map[string]int64)} + for _, table := range []string{ + "logical_file", "physical_file", "file_chunk", "chunk", "blocks", + "storage_blocks", "chunk_block_refs", "container", "snapshot", + "snapshot_path", "snapshot_file", + } { + fingerprint.tableCounts[table] = mutationBackendInt64( + t, dbconn, "SELECT COUNT(*) FROM "+table, + ) + } + fingerprint.logical = mutationBackendRows(t, dbconn, ` + SELECT file_hash, original_name, total_size, status, ref_count, + retry_count, chunker_version + FROM logical_file + ORDER BY file_hash, total_size`) + fingerprint.physical = mutationBackendRows(t, dbconn, ` + SELECT pf.path, lf.file_hash, COALESCE(pf.mode, -1), + COALESCE(pf.uid, -1), COALESCE(pf.gid, -1), + pf.is_metadata_complete + FROM physical_file pf + JOIN logical_file lf ON lf.id = pf.logical_file_id + ORDER BY pf.path, lf.file_hash`) + fingerprint.fileChunks = mutationBackendRows(t, dbconn, ` + SELECT lf.file_hash, fc.chunk_order, c.chunk_hash, c.size + FROM file_chunk fc + JOIN logical_file lf ON lf.id = fc.logical_file_id + JOIN chunk c ON c.id = fc.chunk_id + ORDER BY lf.file_hash, fc.chunk_order, c.chunk_hash`) + fingerprint.chunks = mutationBackendRows(t, dbconn, ` + SELECT chunk_hash, size, status, live_ref_count, pin_count, + retry_count, chunker_version + FROM chunk + ORDER BY chunk_hash, size`) + fingerprint.legacy = mutationBackendRows(t, dbconn, ` + SELECT c.chunk_hash, b.codec, b.format_version, b.plaintext_size, + b.stored_size, co.filename, b.block_offset + FROM blocks b + JOIN chunk c ON c.id = b.chunk_id + JOIN container co ON co.id = b.container_id + ORDER BY c.chunk_hash, co.filename, b.block_offset`) + fingerprint.packed = mutationBackendRows(t, dbconn, ` + SELECT co.filename, sb.container_offset, sb.format_version, sb.codec, + sb.plaintext_size, sb.compression_codec, + COALESCE(sb.compression_level, -1), sb.stored_size, + sb.block_hash + FROM storage_blocks sb + JOIN container co ON co.id = sb.container_id + ORDER BY co.filename, sb.container_offset`) + fingerprint.packedRefs = mutationBackendRows(t, dbconn, ` + SELECT c.chunk_hash, co.filename, sb.container_offset, + cbr.offset_in_block, cbr.size_in_block + FROM chunk_block_refs cbr + JOIN chunk c ON c.id = cbr.chunk_id + JOIN storage_blocks sb ON sb.id = cbr.block_id + JOIN container co ON co.id = sb.container_id + ORDER BY c.chunk_hash, co.filename, sb.container_offset`) + fingerprint.containers = mutationBackendRows(t, dbconn, ` + SELECT filename, sealed, sealing, quarantine, current_size, max_size, + COALESCE(container_hash, '') + FROM container + ORDER BY filename`) + fingerprint.snapshots = mutationBackendRows(t, dbconn, ` + SELECT id, type, COALESCE(label, ''), COALESCE(parent_id, '') + FROM snapshot + ORDER BY id`) + fingerprint.snapshotPaths = mutationBackendRows(t, dbconn, ` + SELECT path + FROM snapshot_path + ORDER BY path`) + fingerprint.members = mutationBackendRows(t, dbconn, ` + SELECT sf.snapshot_id, sp.path, lf.file_hash, + COALESCE(sf.size, -1), COALESCE(sf.mode, -1) + FROM snapshot_file sf + JOIN snapshot_path sp ON sp.id = sf.path_id + JOIN logical_file lf ON lf.id = sf.logical_file_id + ORDER BY sf.snapshot_id, sp.path, lf.file_hash`) + fingerprint.files = mutationFileManifest(t, containerDir) + return fingerprint +} + +func assertMutationFingerprintEqual( + t *testing.T, + before, after mutationRepositoryFingerprint, +) { + t.Helper() + if !reflect.DeepEqual(before, after) { + t.Fatalf("repository/container state changed unexpectedly:\nbefore=%+v\nafter=%+v", before, after) + } +} + +func mutationBackendRows(t *testing.T, dbconn *sql.DB, query string, args ...any) []string { + t.Helper() + rows, err := dbconn.QueryContext(context.Background(), query, args...) + if err != nil { + t.Fatalf("query mutation fingerprint: %v\nquery: %s", err, query) + } + defer func() { _ = rows.Close() }() + columns, err := rows.Columns() + if err != nil { + t.Fatalf("mutation fingerprint columns: %v", err) + } + result := make([]string, 0) + for rows.Next() { + values := make([]any, len(columns)) + destinations := make([]any, len(columns)) + for i := range values { + destinations[i] = &values[i] + } + if err := rows.Scan(destinations...); err != nil { + t.Fatalf("scan mutation fingerprint row: %v", err) + } + parts := make([]string, len(values)) + for i, value := range values { + switch typed := value.(type) { + case []byte: + parts[i] = hex.EncodeToString(typed) + default: + parts[i] = fmt.Sprint(typed) + } + } + result = append(result, strings.Join(parts, "|")) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate mutation fingerprint rows: %v", err) + } + return result +} + +func mutationBackendInt64(t *testing.T, dbconn *sql.DB, query string, args ...any) int64 { + t.Helper() + var value int64 + if err := dbconn.QueryRowContext(context.Background(), query, args...).Scan(&value); err != nil { + t.Fatalf("query mutation integer: %v\nquery: %s", err, query) + } + return value +} + +func mutationBackendString(t *testing.T, dbconn *sql.DB, query string, args ...any) string { + t.Helper() + var value string + if err := dbconn.QueryRowContext(context.Background(), query, args...).Scan(&value); err != nil { + t.Fatalf("query mutation string: %v\nquery: %s", err, query) + } + return value +} + +func mutationFileManifest(t *testing.T, root string) []string { + t.Helper() + manifest := make([]string, 0) + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + sum := sha256.Sum256(data) + manifest = append(manifest, fmt.Sprintf( + "%s|%d|%s", + filepath.ToSlash(relative), + len(data), + hex.EncodeToString(sum[:]), + )) + return nil + }) + if err != nil { + t.Fatalf("capture file manifest under %q: %v", root, err) + } + sort.Strings(manifest) + return manifest +} + +func assertMutationFile(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + t.Fatalf("read mutation output %q: %v", path, err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("mutation output %q mismatch: got %q want %q", path, got, want) + } +} + +func mutationManifestEntry(relativePath string, payload []byte) string { + sum := sha256.Sum256(payload) + return fmt.Sprintf( + "%s|%d|%s", + filepath.ToSlash(relativePath), + len(payload), + hex.EncodeToString(sum[:]), + ) +} + +func assertMutationFileManifest(t *testing.T, root string, want []string) { + t.Helper() + got := mutationFileManifest(t, root) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Fatalf("destination manifest mismatch under %q: got %v want %v", root, got, want) + } +} + +func assertStoreResultShape(t *testing.T, result engine.StoreResult) { + t.Helper() + if result.SourcePath == "" || result.StoredPath == "" || + result.LogicalFileID <= 0 || result.FileHash == "" { + t.Fatalf("Store returned incomplete stable fields: %+v", result) + } + if result.PhysicalFileID != 0 || result.BytesLogical != 0 || + result.BytesStored != 0 || result.ChunksCreated != 0 || + result.ChunksReused != 0 || result.Warnings != nil { + t.Fatalf("Store dormant result fields changed: %+v", result) + } +} + +func seedMutationDeadContainer( + t *testing.T, + dbconn *sql.DB, + containerDir, filename string, + payload []byte, +) { + t.Helper() + if err := os.MkdirAll(containerDir, 0o700); err != nil { + t.Fatalf("create dead-container directory: %v", err) + } + if err := os.WriteFile(filepath.Join(containerDir, filename), payload, 0o600); err != nil { + t.Fatalf("write dead-container fixture: %v", err) + } + var containerID int64 + if err := dbconn.QueryRowContext(context.Background(), ` + INSERT INTO container + (filename, current_size, max_size, sealed, quarantine) + VALUES ($1, $2, $3, TRUE, FALSE) + RETURNING id`, + filename, int64(len(payload)), container.GetContainerMaxSize(), + ).Scan(&containerID); err != nil { + t.Fatalf("insert dead container: %v", err) + } + sum := sha256.Sum256(payload) + var chunkID int64 + if err := dbconn.QueryRowContext(context.Background(), ` + INSERT INTO chunk + (chunk_hash, size, status, live_ref_count, pin_count, chunker_version) + VALUES ($1, $2, 'COMPLETED', 0, 0, 'v2-fastcdc') + RETURNING id`, + hex.EncodeToString(sum[:]), int64(len(payload)), + ).Scan(&chunkID); err != nil { + t.Fatalf("insert dead chunk: %v", err) + } + if _, err := dbconn.ExecContext(context.Background(), ` + INSERT INTO blocks + (chunk_id, codec, format_version, plaintext_size, stored_size, + container_id, block_offset) + VALUES ($1, 'plain', 1, $2, $2, $3, 0)`, + chunkID, int64(len(payload)), containerID, + ); err != nil { + t.Fatalf("insert dead block: %v", err) + } +} diff --git a/internal/engine/mutation_backend_contract_test.go b/internal/engine/mutation_backend_contract_test.go new file mode 100644 index 00000000..9e818293 --- /dev/null +++ b/internal/engine/mutation_backend_contract_test.go @@ -0,0 +1,823 @@ +package engine_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/invariants" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineMutationStoreRemoveAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newMutationBackendFixture(t, backend) + payloadA := []byte("phase9 deterministic payload A") + payloadB := []byte("phase9 replacement payload B") + + storedA := fixture.store(t, "docs/a.txt", payloadA) + storedB := fixture.store(t, "docs/b.txt", payloadA) + storedC := fixture.store(t, "docs/c.txt", payloadA) + for _, result := range []engine.StoreResult{storedA, storedB, storedC} { + assertStoreResultShape(t, result) + } + if storedA.AlreadyStored || !storedB.AlreadyStored || !storedC.AlreadyStored { + t.Fatalf("unexpected Store dedup flags: A=%+v B=%+v C=%+v", storedA, storedB, storedC) + } + if storedA.LogicalFileID != storedB.LogicalFileID || + storedA.LogicalFileID != storedC.LogicalFileID || + storedA.FileHash != storedB.FileHash || + storedA.FileHash != storedC.FileHash { + t.Fatalf("identical content did not share logical identity: A=%+v B=%+v C=%+v", storedA, storedB, storedC) + } + + if err := os.WriteFile( + filepath.Join(fixture.inputRoot, "docs", "b.txt"), + payloadB, + 0o600, + ); err != nil { + t.Fatalf("rewrite replacement source: %v", err) + } + replacement := fixture.storeExisting(t, "docs/b.txt") + assertStoreResultShape(t, replacement) + if replacement.AlreadyStored || replacement.LogicalFileID == storedA.LogicalFileID || + replacement.FileHash == storedA.FileHash { + t.Fatalf("replacement did not retarget the physical path: %+v", replacement) + } + empty := fixture.store(t, "empty.txt", nil) + assertStoreResultShape(t, empty) + if empty.AlreadyStored { + t.Fatalf("first empty-file Store unexpectedly deduplicated: %+v", empty) + } + fixture.finalize(t) + + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM file_chunk WHERE logical_file_id = $1`, storedA.LogicalFileID); got == 0 { + t.Fatal("non-empty Store produced no recipe entries") + } + storedAChunkHash := mutationBackendString(t, backend.DB, ` + SELECT c.chunk_hash + FROM file_chunk fc + JOIN chunk c ON c.id = fc.chunk_id + WHERE fc.logical_file_id = $1 + ORDER BY fc.chunk_order + LIMIT 1`, storedA.LogicalFileID) + if got := mutationBackendInt64(t, backend.DB, + `SELECT live_ref_count FROM chunk WHERE chunk_hash = $1`, storedAChunkHash); got != 1 { + t.Fatalf("deduplicated content live_ref_count: got %d want 1", got) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT ref_count FROM logical_file WHERE id = $1`, storedA.LogicalFileID); got != 2 { + t.Fatalf("replacement-adjusted logical ref_count: got %d want 2", got) + } + if got := mutationBackendString(t, backend.DB, ` + SELECT lf.file_hash + FROM physical_file pf + JOIN logical_file lf ON lf.id = pf.logical_file_id + WHERE pf.path = $1`, "docs/b.txt"); got != replacement.FileHash { + t.Fatalf("replacement mapping hash: got %q want %q", got, replacement.FileHash) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM file_chunk WHERE logical_file_id = $1`, empty.LogicalFileID); got != 0 { + t.Fatalf("empty file recipe count: got %d want 0", got) + } + removePath := fixture.useAbsoluteStoredPath(t, "docs/c.txt") + + beforeDryRun := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + dryRun, err := fixture.engine.RemoveStoredPaths(context.Background(), engine.RemoveStoredPathsRequest{ + StoredPaths: []string{removePath, " " + removePath + " ", ""}, + DryRun: true, + }) + if err != nil { + t.Fatalf("RemoveStoredPaths dry-run: %v", err) + } + if dryRun.ExecutionMode != engine.ExecutionModeSequential || + dryRun.Summary != (engine.BatchSummary{OK: 1, Failed: 1, Skipped: 1}) || + len(dryRun.Items) != 3 || + dryRun.Items[0].Status != engine.BatchItemPlanned || + dryRun.Items[1].Status != engine.BatchItemSkipped || + dryRun.Items[2].Status != engine.BatchItemFailed { + t.Fatalf("unexpected stored-path dry-run result: %+v", dryRun) + } + assertMutationFingerprintEqual( + t, + beforeDryRun, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + unlinked, err := fixture.engine.RemoveStoredPaths(context.Background(), engine.RemoveStoredPathsRequest{ + StoredPaths: []string{removePath}, + }) + if err != nil || len(unlinked.Items) != 1 || + unlinked.Items[0].Status != engine.BatchItemOK || + !unlinked.Items[0].MappingRemoved || + unlinked.Items[0].RemainingRefCount != 1 { + t.Fatalf("RemoveStoredPaths live: got (%+v, %v)", unlinked, err) + } + beforeRemoveDryRun := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + removeDryRun, err := fixture.engine.Remove(context.Background(), engine.RemoveRequest{ + FileIDs: []int64{storedA.LogicalFileID}, + DryRun: true, + }) + if err != nil || !removeDryRun.DryRun || + removeDryRun.ExecutionMode != engine.ExecutionModeSequential || + removeDryRun.Summary != (engine.BatchSummary{OK: 1}) || + len(removeDryRun.Items) != 1 || + removeDryRun.Items[0].Status != engine.BatchItemOK || + removeDryRun.Items[0].LogicalFileRemoved { + t.Fatalf("Remove dry-run: got (%+v, %v)", removeDryRun, err) + } + assertMutationFingerprintEqual( + t, + beforeRemoveDryRun, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + containerBeforeRemove := mutationFileManifest(t, fixture.containerDir) + removed, err := fixture.engine.Remove(context.Background(), engine.RemoveRequest{ + FileIDs: []int64{storedA.LogicalFileID}, + }) + if err != nil || len(removed.Items) != 1 || + removed.ExecutionMode != engine.ExecutionModeSequential || + removed.Summary != (engine.BatchSummary{OK: 1}) || + removed.Items[0].Status != engine.BatchItemOK || + !removed.Items[0].LogicalFileRemoved || + removed.Items[0].RemovedChunkAssociations == 0 { + t.Fatalf("Remove by ID: got (%+v, %v)", removed, err) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM logical_file WHERE id = $1`, storedA.LogicalFileID); got != 0 { + t.Fatalf("removed logical file count: got %d want 0", got) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM file_chunk WHERE logical_file_id = $1`, storedA.LogicalFileID); got != 0 { + t.Fatalf("removed logical recipe rows remain: %d", got) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT live_ref_count FROM chunk WHERE chunk_hash = $1`, storedAChunkHash); got != 0 { + t.Fatalf("removed logical chunk live_ref_count: got %d want 0", got) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM physical_file WHERE path IN ($1, $2)`, "docs/a.txt", removePath); got != 0 { + t.Fatalf("removed mappings remain: %d", got) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM physical_file WHERE path = $1`, "docs/b.txt"); got != 1 { + t.Fatalf("replacement mapping was disturbed: %d", got) + } + if after := mutationFileManifest(t, fixture.containerDir); !reflect.DeepEqual(containerBeforeRemove, after) { + t.Fatalf("Remove deleted or rewrote payload containers:\nbefore=%v\nafter=%v", containerBeforeRemove, after) + } + }) +} + +func TestEngineMutationSnapshotLifecycleAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newMutationBackendFixture(t, backend) + fixture.store(t, "docs/a.txt", []byte("snapshot lifecycle A")) + fixture.store(t, "docs/sub/b.txt", []byte("snapshot lifecycle B")) + fixture.store(t, "img/c.txt", []byte("snapshot lifecycle C")) + fixture.finalize(t) + eng := fixture.readEngine(t) + + root, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-root", Label: "root", + }) + if err != nil || root.SnapshotID != "phase9-root" || + root.Type != engine.SnapshotTypeFull || root.FilesInserted != 3 || + root.PathsCount != 0 || root.Label != "root" { + t.Fatalf("SnapshotCreate root: got (%+v, %v)", root, err) + } + child, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-child", Label: "child", ParentID: "phase9-root", + }) + if err != nil || child.SnapshotID != "phase9-child" || + child.Type != engine.SnapshotTypeFull || child.FilesInserted != 3 || + child.ParentID != "phase9-root" { + t.Fatalf("SnapshotCreate child: got (%+v, %v)", child, err) + } + partial, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-partial", Label: "partial", + Paths: []string{"docs/", "docs/a.txt", "docs/"}, + }) + if err != nil || partial.SnapshotID != "phase9-partial" || + partial.Type != engine.SnapshotTypePartial || + partial.PathsCount != 3 || partial.FilesInserted != 2 { + t.Fatalf("SnapshotCreate partial: got (%+v, %v)", partial, err) + } + if got := mutationBackendRows(t, backend.DB, ` + SELECT sp.path + FROM snapshot_file sf + JOIN snapshot_path sp ON sp.id = sf.path_id + WHERE sf.snapshot_id = $1 + ORDER BY sp.path`, "phase9-partial"); !reflect.DeepEqual(got, []string{"docs/a.txt", "docs/sub/b.txt"}) { + t.Fatalf("partial membership: got %v", got) + } + + beforePreview := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + preview, err := eng.SnapshotDelete(context.Background(), engine.SnapshotDeleteRequest{ + SnapshotID: "phase9-root", + Mode: engine.SnapshotDeleteModePreview, + }) + if err != nil || preview.Deleted || preview.Preview == nil || + preview.Mode != engine.SnapshotDeleteModePreview || + preview.Preview.Parent != (engine.SnapshotDeleteParent{State: engine.SnapshotDeleteParentNone}) || + !reflect.DeepEqual(preview.Preview.Children, []string{"phase9-child"}) || + preview.Preview.TotalFiles != 3 || + preview.Preview.UniqueFiles != 0 || + preview.Preview.SharedFiles != 3 { + t.Fatalf("SnapshotDelete preview: got (%+v, %v)", preview, err) + } + assertMutationFingerprintEqual( + t, + beforePreview, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + contentBeforeDelete := mutationBackendRows(t, backend.DB, ` + SELECT lf.file_hash, lf.ref_count, c.chunk_hash, c.live_ref_count + FROM logical_file lf + LEFT JOIN file_chunk fc ON fc.logical_file_id = lf.id + LEFT JOIN chunk c ON c.id = fc.chunk_id + ORDER BY lf.file_hash, fc.chunk_order`) + containerBeforeDelete := mutationFileManifest(t, fixture.containerDir) + deleted, err := eng.SnapshotDelete(context.Background(), engine.SnapshotDeleteRequest{ + SnapshotID: "phase9-root", + Mode: engine.SnapshotDeleteModeExecute, + }) + if err != nil || deleted != (engine.SnapshotDeleteResult{ + SnapshotID: "phase9-root", + Mode: engine.SnapshotDeleteModeExecute, + Deleted: true, + }) { + t.Fatalf("SnapshotDelete execute: got (%+v, %v)", deleted, err) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM snapshot WHERE id = $1`, "phase9-root"); got != 0 { + t.Fatalf("deleted snapshot remains: %d", got) + } + if got := mutationBackendInt64(t, backend.DB, + `SELECT COUNT(*) FROM snapshot_file WHERE snapshot_id = $1`, "phase9-root"); got != 0 { + t.Fatalf("deleted snapshot membership remains: %d", got) + } + var parentID any + if err := backend.DB.QueryRowContext(context.Background(), + `SELECT parent_id FROM snapshot WHERE id = $1`, "phase9-child").Scan(&parentID); err != nil { + t.Fatalf("query child parent after delete: %v", err) + } + if parentID != nil { + t.Fatalf("child parent was not cleared: %v", parentID) + } + contentAfterDelete := mutationBackendRows(t, backend.DB, ` + SELECT lf.file_hash, lf.ref_count, c.chunk_hash, c.live_ref_count + FROM logical_file lf + LEFT JOIN file_chunk fc ON fc.logical_file_id = lf.id + LEFT JOIN chunk c ON c.id = fc.chunk_id + ORDER BY lf.file_hash, fc.chunk_order`) + if !reflect.DeepEqual(contentBeforeDelete, contentAfterDelete) { + t.Fatalf("snapshot delete changed content graph:\nbefore=%v\nafter=%v", contentBeforeDelete, contentAfterDelete) + } + if after := mutationFileManifest(t, fixture.containerDir); !reflect.DeepEqual(containerBeforeDelete, after) { + t.Fatalf("snapshot delete changed containers:\nbefore=%v\nafter=%v", containerBeforeDelete, after) + } + }) +} + +func TestEngineMutationRestoreAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newMutationBackendFixture(t, backend) + payloadA := []byte("phase9 restore payload A") + payloadB := []byte("phase9 restore payload B") + storedA := fixture.store(t, "docs/a.txt", payloadA) + storedB := fixture.store(t, "docs/b.txt", payloadB) + fixture.finalize(t) + eng := fixture.readEngine(t) + if _, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-restore-snapshot", + }); err != nil { + t.Fatalf("create restore snapshot: %v", err) + } + storedPathTarget := fixture.useAbsoluteStoredPath(t, "docs/b.txt") + before := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + + byIDRoot := filepath.Join(t.TempDir(), "by-id") + byID, err := eng.Restore(context.Background(), engine.RestoreRequest{ + FileIDs: []int64{storedA.LogicalFileID}, DestinationRoot: byIDRoot, + Overwrite: true, + }) + wantByIDPath := filepath.Join(byIDRoot, "a.txt") + if err != nil || byID.ExecutionMode != engine.ExecutionModeSequential || + byID.Summary != (engine.BatchSummary{OK: 1}) || + len(byID.Items) != 1 || byID.Items[0].Status != engine.BatchItemOK || + byID.Items[0].DestinationPath != wantByIDPath || + byID.Items[0].RestoredHash != storedA.FileHash { + t.Fatalf("Restore by ID: got (%+v, %v)", byID, err) + } + assertMutationFile(t, wantByIDPath, payloadA) + assertMutationFileManifest(t, byIDRoot, []string{ + mutationManifestEntry("a.txt", payloadA), + }) + + storedPathRoot := filepath.Join(t.TempDir(), "stored-path") + storedPath, err := eng.RestoreStoredPath(context.Background(), engine.RestoreStoredPathRequest{ + StoredPath: storedPathTarget, DestinationMode: engine.RestoreDestinationPrefix, + DestinationRoot: storedPathRoot, Overwrite: true, NoMetadata: true, + }) + wantStoredPath := filepath.Join( + storedPathRoot, + strings.TrimPrefix(filepath.Clean(storedPathTarget), string(filepath.Separator)), + ) + if err != nil || storedPath.StoredPath != storedPathTarget || + storedPath.FileID != storedB.LogicalFileID || + storedPath.DestinationMode != engine.RestoreDestinationPrefix || + storedPath.DestinationPath != wantStoredPath || + storedPath.RestoredHash != storedB.FileHash { + t.Fatalf("RestoreStoredPath: got (%+v, %v)", storedPath, err) + } + assertMutationFile(t, wantStoredPath, payloadB) + storedPathRelative, err := filepath.Rel(storedPathRoot, wantStoredPath) + if err != nil { + t.Fatalf("derive stored-path destination relative path: %v", err) + } + assertMutationFileManifest(t, storedPathRoot, []string{ + mutationManifestEntry(storedPathRelative, payloadB), + }) + + snapshotRoot := filepath.Join(t.TempDir(), "snapshot") + snapshotResult, err := eng.SnapshotRestore(context.Background(), engine.SnapshotRestoreRequest{ + SnapshotID: "phase9-restore-snapshot", + Selection: engine.SnapshotRestoreSelection{ + ExactPaths: []string{"docs/b.txt", "docs/a.txt", "docs/a.txt"}, + Prefixes: []string{"docs/", "docs/"}, + }, + Destination: engine.SnapshotRestoreDestination{ + Mode: engine.SnapshotRestoreDestinationOriginal, + Path: snapshotRoot, + }, + Overwrite: true, + Metadata: engine.SnapshotRestoreMetadataNone, + }) + wantSnapshotPaths := []string{ + filepath.Join(snapshotRoot, "docs", "a.txt"), + filepath.Join(snapshotRoot, "docs", "b.txt"), + } + if err != nil || snapshotResult.SnapshotID != "phase9-restore-snapshot" || + snapshotResult.DestinationMode != engine.SnapshotRestoreDestinationOriginal || + snapshotResult.RequestedPathsCount != 0 || + snapshotResult.RestoredFiles != 2 || + snapshotResult.OutputTarget != snapshotRoot || + !reflect.DeepEqual(snapshotResult.OutputPaths, wantSnapshotPaths) || + len(snapshotResult.Warnings) != 0 { + t.Fatalf("SnapshotRestore: got (%+v, %v)", snapshotResult, err) + } + assertMutationFile(t, wantSnapshotPaths[0], payloadA) + assertMutationFile(t, wantSnapshotPaths[1], payloadB) + assertMutationFileManifest(t, snapshotRoot, []string{ + mutationManifestEntry(filepath.Join("docs", "a.txt"), payloadA), + mutationManifestEntry(filepath.Join("docs", "b.txt"), payloadB), + }) + + assertMutationFingerprintEqual( + t, + before, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + if pins := mutationBackendInt64(t, backend.DB, + `SELECT COALESCE(SUM(pin_count), 0) FROM chunk`); pins != 0 { + t.Fatalf("restore left chunk pins: %d", pins) + } + }) +} + +func TestEngineMutationErrorsAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newMutationBackendFixture(t, backend) + retained := fixture.store(t, "retained.txt", []byte("phase9 retained payload")) + fixture.finalize(t) + eng := fixture.readEngine(t) + if _, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-retained", + }); err != nil { + t.Fatalf("create retention snapshot: %v", err) + } + retainedPath := fixture.useAbsoluteStoredPath(t, "retained.txt") + baseline := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := fixture.engine.Store(cancelled, engine.StoreRequest{ + SourcePath: "retained.txt", Codec: "plain", + }); !errors.Is(err, context.Canceled) { + t.Fatalf("pre-cancelled Store: %v", err) + } + cancelledOperations := []struct { + name string + call func() error + }{ + {"Remove", func() error { + _, err := eng.Remove(cancelled, engine.RemoveRequest{FileIDs: []int64{retained.LogicalFileID}}) + return err + }}, + {"RemoveStoredPaths", func() error { + _, err := eng.RemoveStoredPaths(cancelled, engine.RemoveStoredPathsRequest{StoredPaths: []string{retainedPath}}) + return err + }}, + {"Restore", func() error { + _, err := eng.Restore(cancelled, engine.RestoreRequest{ + FileIDs: []int64{retained.LogicalFileID}, DestinationRoot: t.TempDir(), + }) + return err + }}, + {"RestoreStoredPath", func() error { + _, err := eng.RestoreStoredPath(cancelled, engine.RestoreStoredPathRequest{ + StoredPath: retainedPath, DestinationMode: engine.RestoreDestinationPrefix, + DestinationRoot: t.TempDir(), + }) + return err + }}, + {"SnapshotCreate", func() error { + _, err := eng.SnapshotCreate(cancelled, engine.SnapshotCreateRequest{ID: "phase9-cancelled"}) + return err + }}, + {"SnapshotDelete", func() error { + _, err := eng.SnapshotDelete(cancelled, engine.SnapshotDeleteRequest{ + SnapshotID: "phase9-retained", Mode: engine.SnapshotDeleteModePreview, + }) + return err + }}, + {"SnapshotRestore", func() error { + _, err := eng.SnapshotRestore(cancelled, engine.SnapshotRestoreRequest{ + SnapshotID: "phase9-retained", + Destination: engine.SnapshotRestoreDestination{ + Mode: engine.SnapshotRestoreDestinationOriginal, + Path: t.TempDir(), + }, + }) + return err + }}, + } + for _, operation := range cancelledOperations { + t.Run("pre-cancelled "+operation.name, func(t *testing.T) { + if err := operation.call(); !errors.Is(err, context.Canceled) { + t.Fatalf("%s: expected context cancellation, got %v", operation.name, err) + } + }) + } + if _, err := fixture.engine.Store(context.Background(), engine.StoreRequest{ + SourcePath: "retained.txt", Recursive: true, + }); !errors.Is(err, engine.ErrNotImplemented) || !engine.IsUnsupported(err) { + t.Fatalf("recursive Store classification: %v", err) + } + if _, err := fixture.engine.Store(context.Background(), engine.StoreRequest{ + SourcePath: " ", + }); err == nil || engine.IsUnsupported(err) { + t.Fatalf("blank-path Store classification: %v", err) + } + if _, err := fixture.engine.Store(context.Background(), engine.StoreRequest{ + SourcePath: "phase9-missing-source.txt", Codec: "plain", + }); err == nil || engine.IsUnsupported(err) { + t.Fatalf("missing-source Store classification: %v", err) + } + if _, err := fixture.engine.Store(context.Background(), engine.StoreRequest{ + SourcePath: "retained.txt", Codec: "invalid-codec", + }); err == nil || engine.IsUnsupported(err) { + t.Fatalf("invalid-codec Store classification: %v", err) + } + assertMutationFingerprintEqual( + t, + baseline, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + remove, err := eng.Remove(context.Background(), engine.RemoveRequest{ + FileIDs: []int64{retained.LogicalFileID}, + }) + if err != nil || len(remove.Items) != 1 || + remove.Items[0].Status != engine.BatchItemFailed || + remove.Items[0].InvariantCode != invariants.CodeSnapshotRetainedDeleteBlocked || + remove.Items[0].RecommendedAction == "" { + t.Fatalf("snapshot-retained Remove: got (%+v, %v)", remove, err) + } + unlink, err := eng.RemoveStoredPaths(context.Background(), engine.RemoveStoredPathsRequest{ + StoredPaths: []string{retainedPath}, + }) + if err != nil || len(unlink.Items) != 1 || + unlink.Items[0].Status != engine.BatchItemFailed || + unlink.Items[0].InvariantCode != invariants.CodeSnapshotRetainedDeleteBlocked || + unlink.Items[0].RecommendedAction == "" { + t.Fatalf("snapshot-retained RemoveStoredPaths: got (%+v, %v)", unlink, err) + } + assertMutationFingerprintEqual( + t, + baseline, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + for _, operation := range []struct { + name string + call func() error + want string + }{ + { + name: "duplicate snapshot", + call: func() error { + _, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ID: "phase9-retained"}) + return err + }, + want: "insert snapshot id=phase9-retained", + }, + { + name: "missing snapshot parent", + call: func() error { + _, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-orphan", ParentID: "missing-parent", + }) + return err + }, + want: "parent snapshot", + }, + { + name: "missing snapshot create path", + call: func() error { + _, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-missing-path", Paths: []string{"missing.txt"}, + }) + return err + }, + want: "path not found", + }, + { + name: "missing snapshot delete", + call: func() error { + _, err := eng.SnapshotDelete(context.Background(), engine.SnapshotDeleteRequest{ + SnapshotID: "missing-snapshot", Mode: engine.SnapshotDeleteModeExecute, + }) + return err + }, + want: "not found", + }, + { + name: "invalid snapshot delete mode", + call: func() error { + _, err := eng.SnapshotDelete(context.Background(), engine.SnapshotDeleteRequest{ + SnapshotID: "phase9-retained", Mode: engine.SnapshotDeleteMode("invalid"), + }) + return err + }, + want: "unknown snapshot delete mode", + }, + { + name: "missing snapshot restore", + call: func() error { + _, err := eng.SnapshotRestore(context.Background(), engine.SnapshotRestoreRequest{ + SnapshotID: "missing-snapshot", + Destination: engine.SnapshotRestoreDestination{ + Mode: engine.SnapshotRestoreDestinationOriginal, + Path: filepath.Join(t.TempDir(), "missing-snapshot"), + }, + }) + return err + }, + want: "not found", + }, + { + name: "missing snapshot restore path", + call: func() error { + _, err := eng.SnapshotRestore(context.Background(), engine.SnapshotRestoreRequest{ + SnapshotID: "phase9-retained", + Paths: []string{"missing.txt"}, + Destination: engine.SnapshotRestoreDestination{ + Mode: engine.SnapshotRestoreDestinationOriginal, + Path: filepath.Join(t.TempDir(), "missing-path"), + }, + }) + return err + }, + want: "path not found", + }, + { + name: "invalid snapshot restore regex", + call: func() error { + _, err := eng.SnapshotRestore(context.Background(), engine.SnapshotRestoreRequest{ + SnapshotID: "phase9-retained", + Selection: engine.SnapshotRestoreSelection{Regex: "("}, + Destination: engine.SnapshotRestoreDestination{ + Mode: engine.SnapshotRestoreDestinationOriginal, + Path: filepath.Join(t.TempDir(), "invalid-regex"), + }, + }) + return err + }, + want: "invalid snapshot restore regex", + }, + } { + t.Run(operation.name, func(t *testing.T) { + err := operation.call() + if err == nil || !strings.Contains(err.Error(), operation.want) || + engine.IsUnsupported(err) { + t.Fatalf("expected stable error containing %q, got %v", operation.want, err) + } + }) + } + assertMutationFingerprintEqual( + t, + baseline, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + missingRestore, err := eng.Restore(context.Background(), engine.RestoreRequest{ + FileIDs: []int64{-1}, DestinationRoot: filepath.Join(t.TempDir(), "missing"), + }) + if err != nil || len(missingRestore.Items) != 1 || + missingRestore.Items[0].Status != engine.BatchItemFailed || + missingRestore.Summary != (engine.BatchSummary{Failed: 1}) { + t.Fatalf("missing Restore item: got (%+v, %v)", missingRestore, err) + } + failFastRestore, err := eng.Restore(context.Background(), engine.RestoreRequest{ + FileIDs: []int64{-1, retained.LogicalFileID}, DestinationRoot: filepath.Join(t.TempDir(), "fail-fast"), + FailFast: true, + }) + if err != nil || len(failFastRestore.Items) != 1 || + failFastRestore.Items[0].Status != engine.BatchItemFailed || + failFastRestore.Summary != (engine.BatchSummary{Failed: 1, Skipped: 1}) { + t.Fatalf("fail-fast Restore: got (%+v, %v)", failFastRestore, err) + } + failFastRemove, err := eng.Remove(context.Background(), engine.RemoveRequest{ + FileIDs: []int64{-1, retained.LogicalFileID}, FailFast: true, + }) + if err != nil || len(failFastRemove.Items) != 1 || + failFastRemove.Items[0].Status != engine.BatchItemFailed || + failFastRemove.Summary != (engine.BatchSummary{Failed: 1, Skipped: 1}) { + t.Fatalf("fail-fast Remove: got (%+v, %v)", failFastRemove, err) + } + if result, err := eng.RestoreStoredPath(context.Background(), engine.RestoreStoredPathRequest{ + StoredPath: "missing.txt", DestinationMode: engine.RestoreDestinationPrefix, + DestinationRoot: filepath.Join(t.TempDir(), "missing-path"), + NoMetadata: true, + }); err == nil || result != (engine.RestoreStoredPathResult{}) { + t.Fatalf("missing RestoreStoredPath: got (%+v, %v)", result, err) + } + collisionRoot := filepath.Join(t.TempDir(), "collision") + if err := os.MkdirAll(collisionRoot, 0o700); err != nil { + t.Fatalf("create collision root: %v", err) + } + collisionPath := filepath.Join(collisionRoot, "retained.txt") + collisionBytes := []byte("preserve collision") + if err := os.WriteFile(collisionPath, collisionBytes, 0o600); err != nil { + t.Fatalf("write collision fixture: %v", err) + } + collision, err := eng.Restore(context.Background(), engine.RestoreRequest{ + FileIDs: []int64{retained.LogicalFileID}, DestinationRoot: collisionRoot, + }) + if err != nil || len(collision.Items) != 1 || + collision.Items[0].Status != engine.BatchItemFailed || + collision.Summary != (engine.BatchSummary{Failed: 1}) { + t.Fatalf("Restore collision: got (%+v, %v)", collision, err) + } + assertMutationFile(t, collisionPath, collisionBytes) + assertMutationFileManifest(t, collisionRoot, []string{ + mutationManifestEntry("retained.txt", collisionBytes), + }) + + snapshotCollisionRoot := filepath.Join(t.TempDir(), "snapshot-collision") + if err := os.MkdirAll(snapshotCollisionRoot, 0o700); err != nil { + t.Fatalf("create snapshot collision root: %v", err) + } + snapshotCollisionPath := filepath.Join(snapshotCollisionRoot, "retained.txt") + snapshotCollisionBytes := []byte("preserve snapshot collision") + if err := os.WriteFile(snapshotCollisionPath, snapshotCollisionBytes, 0o600); err != nil { + t.Fatalf("write snapshot collision fixture: %v", err) + } + if result, err := eng.SnapshotRestore(context.Background(), engine.SnapshotRestoreRequest{ + SnapshotID: "phase9-retained", + Destination: engine.SnapshotRestoreDestination{ + Mode: engine.SnapshotRestoreDestinationOriginal, + Path: snapshotCollisionRoot, + }, + Metadata: engine.SnapshotRestoreMetadataNone, + }); err == nil || !reflect.DeepEqual(result, engine.SnapshotRestoreResult{}) { + t.Fatalf("SnapshotRestore collision: got (%+v, %v)", result, err) + } + assertMutationFile(t, snapshotCollisionPath, snapshotCollisionBytes) + assertMutationFileManifest(t, snapshotCollisionRoot, []string{ + mutationManifestEntry("retained.txt", snapshotCollisionBytes), + }) + + partialRoot := filepath.Join(t.TempDir(), "partial") + partial, err := eng.Restore(context.Background(), engine.RestoreRequest{ + FileIDs: []int64{retained.LogicalFileID, -1}, DestinationRoot: partialRoot, + }) + if err != nil || len(partial.Items) != 2 || + partial.Items[0].Status != engine.BatchItemOK || + partial.Items[1].Status != engine.BatchItemFailed || + partial.Summary != (engine.BatchSummary{OK: 1, Failed: 1}) { + t.Fatalf("partial Restore batch: got (%+v, %v)", partial, err) + } + assertMutationFile(t, filepath.Join(partialRoot, "retained.txt"), []byte("phase9 retained payload")) + assertMutationFileManifest(t, partialRoot, []string{ + mutationManifestEntry("retained.txt", []byte("phase9 retained payload")), + }) + assertMutationFingerprintEqual( + t, + baseline, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + fixture.restartWriter(t) + mismatch := fixture.store(t, "mismatch.txt", []byte("phase9 mismatch payload")) + fixture.finalize(t) + mismatchPath := fixture.useAbsoluteStoredPath(t, "mismatch.txt") + if _, err := backend.DB.ExecContext(context.Background(), + `UPDATE logical_file SET ref_count = $1 WHERE id = $2`, + 5, mismatch.LogicalFileID); err != nil { + t.Fatalf("inject ref-count mismatch: %v", err) + } + mismatchBaseline := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + mismatchResult, err := fixture.readEngine(t).RemoveStoredPaths( + context.Background(), + engine.RemoveStoredPathsRequest{StoredPaths: []string{mismatchPath}}, + ) + if err != nil || len(mismatchResult.Items) != 1 || + mismatchResult.Items[0].Status != engine.BatchItemFailed || + mismatchResult.Items[0].InvariantCode != invariants.CodePhysicalGraphRefCountMismatch { + t.Fatalf("ref-count mismatch rollback: got (%+v, %v)", mismatchResult, err) + } + assertMutationFingerprintEqual( + t, + mismatchBaseline, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + }) +} + +func TestEngineGCDryRunAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newMutationBackendFixture(t, backend) + fixture.store(t, "live.txt", []byte("phase9 live GC payload")) + fixture.finalize(t) + eng := fixture.readEngine(t) + if _, err := eng.SnapshotCreate(context.Background(), engine.SnapshotCreateRequest{ + ID: "phase9-gc-live", + }); err != nil { + t.Fatalf("create GC reachability snapshot: %v", err) + } + const deadFilename = "phase9-gc-dead.bin" + seedMutationDeadContainer( + t, + backend.DB, + fixture.containerDir, + deadFilename, + []byte("phase9 fixed dead GC payload"), + ) + backend.DB.SetMaxOpenConns(4) + before := captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir) + + first, err := eng.GarbageCollect(context.Background(), engine.GarbageCollectRequest{ + DryRun: true, Workers: 7, + }) + if err != nil { + t.Fatalf("GarbageCollect dry-run: %v", err) + } + if !first.DryRun || first.AffectedContainers != 1 || + !reflect.DeepEqual(first.ContainerFilenames, []string{deadFilename}) || + first.SnapshotRetainedContainers != 0 || + first.SnapshotRetainedLogicalFiles != 1 || + first.CurrentOnlyRetainedLogicalFiles != 0 || + first.SnapshotOnlyRetainedLogicalFiles != 0 || + first.SharedRetainedLogicalFiles != 1 || + first.BytesReclaimed != 0 || first.Warnings != nil { + t.Fatalf("unexpected GC dry-run result: %+v", first) + } + assertMutationFingerprintEqual( + t, + before, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + second, err := eng.GarbageCollect(context.Background(), engine.GarbageCollectRequest{DryRun: true}) + if err != nil || !reflect.DeepEqual(first, second) { + t.Fatalf("repeated GC dry-run: first=%+v second=%+v err=%v", first, second, err) + } + assertMutationFingerprintEqual( + t, + before, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := eng.GarbageCollect(cancelled, engine.GarbageCollectRequest{ + DryRun: true, + }); !errors.Is(err, context.Canceled) { + t.Fatalf("pre-cancelled GC dry-run: %v", err) + } + assertMutationFingerprintEqual( + t, + before, + captureMutationRepositoryFingerprint(t, backend.DB, fixture.containerDir), + ) + }) +} diff --git a/internal/engine/read_side_backend_contract_helpers_test.go b/internal/engine/read_side_backend_contract_helpers_test.go new file mode 100644 index 00000000..f45e6276 --- /dev/null +++ b/internal/engine/read_side_backend_contract_helpers_test.go @@ -0,0 +1,218 @@ +package engine_test + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/blocks" + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +var engineReadFixtureTime = time.Date(2026, time.February, 3, 4, 5, 6, 0, time.UTC) + +type engineReadFixture struct { + backend backendtest.Backend + engine *engine.DefaultEngine + containerDir string + logicalA int64 + logicalB int64 + chunkA int64 + containerID int64 +} + +func newEngineReadFixture(t *testing.T, backend backendtest.Backend) engineReadFixture { + t.Helper() + containerDir := filepath.Join(t.TempDir(), "containers") + writer := container.NewLocalWriterWithDirAndDB(containerDir, container.GetContainerMaxSize(), backend.DB) + storageContext := storage.StorageContext{DB: backend.DB, Writer: writer, ContainerDir: containerDir} + t.Cleanup(func() { _ = storageContext.Close() }) + + storedA := storeEngineReadFixtureFile(t, storageContext, "phase7-alpha.txt", "phase7 deterministic alpha payload") + storedB := storeEngineReadFixtureFile(t, storageContext, "phase7-beta.txt", "phase7 deterministic beta payload") + if err := writer.FinalizeContainer(); err != nil { + t.Fatalf("finalize Phase 7 fixture container: %v", err) + } + + chunkA := requireEngineReadInt64(t, backend.DB, + `SELECT chunk_id FROM file_chunk WHERE logical_file_id = $1 ORDER BY chunk_order LIMIT 1`, storedA.FileID) + containerID := requireEngineReadInt64(t, backend.DB, + `SELECT container_id FROM blocks WHERE chunk_id = $1`, chunkA) + + seedEngineReadSnapshots(t, backend.DB, storedA.FileID, storedB.FileID) + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: containerDir}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + return engineReadFixture{ + backend: backend, engine: eng, containerDir: containerDir, + logicalA: storedA.FileID, logicalB: storedB.FileID, + chunkA: chunkA, containerID: containerID, + } +} + +func storeEngineReadFixtureFile(t *testing.T, storageContext storage.StorageContext, name, payload string) storage.StoreFileResult { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(payload), 0o600); err != nil { + t.Fatalf("write fixture input %q: %v", name, err) + } + stored, err := storage.StoreFileWithStorageContextAndCodecResult(storageContext, path, blocks.CodecPlain) + if err != nil { + t.Fatalf("store fixture input %q: %v", name, err) + } + return stored +} + +func seedEngineReadSnapshots(t *testing.T, dbconn *sql.DB, logicalA, logicalB int64) { + t.Helper() + exec := func(query string, args ...any) { + t.Helper() + if _, err := dbconn.ExecContext(context.Background(), query, args...); err != nil { + t.Fatalf("seed engine read fixture: %v\nquery: %s", err, query) + } + } + for _, row := range []struct { + id, kind string + created time.Time + label any + parent any + }{ + {"snap-root", "full", engineReadFixtureTime, nil, nil}, + {"snap-base", "full", engineReadFixtureTime.Add(time.Minute), "base", "snap-root"}, + {"snap-target", "full", engineReadFixtureTime.Add(time.Minute), "target", "snap-base"}, + } { + exec(`INSERT INTO snapshot (id, created_at, type, label, parent_id) VALUES ($1, $2, $3, $4, $5)`, + row.id, row.created, row.kind, row.label, row.parent) + } + paths := []string{"docs/common.txt", "docs/removed.txt", "docs/added.txt"} + for _, path := range paths { + exec(`INSERT INTO snapshot_path (path) VALUES ($1)`, path) + } + pathID := func(path string) int64 { + return requireEngineReadInt64(t, dbconn, `SELECT id FROM snapshot_path WHERE path = $1`, path) + } + insert := func(snapshotID, path string, logicalID int64) { + exec(`INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id, size, mode, mtime) + VALUES ($1, $2, $3, $4, $5, $6)`, + snapshotID, pathID(path), logicalID, 10, 0o644, engineReadFixtureTime) + } + insert("snap-base", "docs/common.txt", logicalA) + insert("snap-base", "docs/removed.txt", logicalA) + insert("snap-target", "docs/common.txt", logicalA) + insert("snap-target", "docs/added.txt", logicalB) +} + +func requireEngineReadInt64(t *testing.T, dbconn *sql.DB, query string, args ...any) int64 { + t.Helper() + var value int64 + if err := dbconn.QueryRowContext(context.Background(), query, args...).Scan(&value); err != nil { + t.Fatalf("query fixture integer: %v\nquery: %s", err, query) + } + return value +} + +type engineReadState struct { + tables map[string]int64 + logical []string + chunks []string + containers []string + files []string +} + +func captureEngineReadState(t *testing.T, dbconn *sql.DB, containerDir string) engineReadState { + t.Helper() + state := engineReadState{tables: map[string]int64{}} + for _, table := range []string{"logical_file", "physical_file", "snapshot", "snapshot_path", "snapshot_file", "chunk", "file_chunk", "blocks", "container", "storage_blocks", "chunk_block_refs"} { + state.tables[table] = requireEngineReadInt64(t, dbconn, "SELECT COUNT(*) FROM "+table) + } + state.logical = engineReadRows(t, dbconn, `SELECT id, status, ref_count, retry_count, chunker_version, updated_at FROM logical_file ORDER BY id`) + state.chunks = engineReadRows(t, dbconn, `SELECT id, status, live_ref_count, pin_count, retry_count, chunker_version, updated_at FROM chunk ORDER BY id`) + state.containers = engineReadRows(t, dbconn, `SELECT id, filename, sealed, sealing, quarantine, current_size, max_size, updated_at FROM container ORDER BY id`) + state.files = engineReadFileManifest(t, containerDir) + return state +} + +func engineReadRows(t *testing.T, dbconn *sql.DB, query string) []string { + t.Helper() + rows, err := dbconn.QueryContext(context.Background(), query) + if err != nil { + t.Fatalf("capture engine read state: %v", err) + } + defer func() { _ = rows.Close() }() + columns, err := rows.Columns() + if err != nil { + t.Fatalf("capture engine read state columns: %v", err) + } + result := make([]string, 0) + for rows.Next() { + values := make([]any, len(columns)) + scans := make([]any, len(columns)) + for i := range values { + scans[i] = &values[i] + } + if err := rows.Scan(scans...); err != nil { + t.Fatalf("capture engine read state row: %v", err) + } + parts := make([]string, len(values)) + for i, value := range values { + parts[i] = fmt.Sprint(value) + } + result = append(result, strings.Join(parts, "|")) + } + if err := rows.Err(); err != nil { + t.Fatalf("capture engine read state iteration: %v", err) + } + return result +} + +func engineReadFileManifest(t *testing.T, root string) []string { + t.Helper() + entries := make([]string, 0) + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + hash := sha256.Sum256(data) + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + entries = append(entries, rel+"|"+hex.EncodeToString(hash[:])) + return nil + }) + if err != nil { + t.Fatalf("capture container manifest: %v", err) + } + sort.Strings(entries) + return entries +} + +func assertEngineReadStateUnchanged(t *testing.T, before, after engineReadState) { + t.Helper() + if fmt.Sprint(before.tables) != fmt.Sprint(after.tables) || + fmt.Sprint(before.logical) != fmt.Sprint(after.logical) || + fmt.Sprint(before.chunks) != fmt.Sprint(after.chunks) || + fmt.Sprint(before.containers) != fmt.Sprint(after.containers) || + fmt.Sprint(before.files) != fmt.Sprint(after.files) { + t.Fatalf("engine read mutated repository state:\n before=%+v\n after=%+v", before, after) + } +} diff --git a/internal/engine/read_side_backend_contract_test.go b/internal/engine/read_side_backend_contract_test.go new file mode 100644 index 00000000..9bead733 --- /dev/null +++ b/internal/engine/read_side_backend_contract_test.go @@ -0,0 +1,226 @@ +package engine_test + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/catalog" + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/observability" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineReadStatsAndInspectAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newEngineReadFixture(t, backend) + before := captureEngineReadState(t, backend.DB, fixture.containerDir) + + stats, err := fixture.engine.Stats(context.Background(), engine.StatsRequest{IncludeContainers: true}) + if err != nil { + t.Fatalf("Stats: %v", err) + } + if stats.Raw == nil || stats.Raw.Logical.TotalFiles != 2 || len(stats.Raw.Containers.Records) != 1 || stats.Raw.Snapshots.TotalSnapshots != 3 { + t.Fatalf("Stats result: %+v", stats.Raw) + } + if again, err := fixture.engine.Stats(context.Background(), engine.StatsRequest{IncludeContainers: true}); err != nil || !equivalentStats(stats.Raw, again.Raw) { + t.Fatalf("repeated Stats: got (%+v, %v)", again.Raw, err) + } + + assertInspectSummary(t, fixture, observability.EntityRepository, "", "total_snapshots", int64(3)) + assertInspectSummary(t, fixture, observability.EntityLogicalFile, fmt.Sprint(fixture.logicalA), "file_id", fixture.logicalA) + assertInspectSummary(t, fixture, observability.EntityChunk, fmt.Sprint(fixture.chunkA), "chunk_id", fixture.chunkA) + assertInspectSummary(t, fixture, observability.EntityContainer, fmt.Sprint(fixture.containerID), "container_id", fixture.containerID) + assertInspectSummary(t, fixture, observability.EntitySnapshot, "snap-target", "snapshot_id", "snap-target") + + withRelations, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{ + Entity: observability.EntitySnapshot, EntityID: "snap-target", + Options: observability.InspectOptions{Relations: true, Deep: true, Limit: 10}, + }) + if err != nil || withRelations.Raw == nil || !relationsSorted(withRelations.Raw.Relations) { + t.Fatalf("Inspect snapshot relations: got (%+v, %v)", withRelations.Raw, err) + } + + _, err = fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: observability.EntityPhysicalFile, EntityID: "1"}) + if !errors.Is(err, observability.ErrUnsupportedEntity) || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + t.Fatalf("physical-file inspect classification: %v", err) + } + assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) + }) +} + +func TestEngineReadSnapshotViewsAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newEngineReadFixture(t, backend) + before := captureEngineReadState(t, backend.DB, fixture.containerDir) + + list, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{Limit: 2}) + if err != nil || list.Count != 2 || !reflect.DeepEqual(snapshotIDs(list), []string{"snap-target", "snap-base"}) { + t.Fatalf("SnapshotList: got (%+v, %v)", list, err) + } + filtered, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{Type: engine.SnapshotTypeFull, Label: "base"}) + if err != nil || !reflect.DeepEqual(snapshotIDs(filtered), []string{"snap-base"}) { + t.Fatalf("SnapshotList filtered: got (%+v, %v)", filtered, err) + } + + show, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "snap-target"}) + if err != nil || show.Snapshot.ParentID != "snap-base" || show.MatchedFileCount != 2 || show.TotalFileCount != 2 || !reflect.DeepEqual(snapshotPaths(show), []string{"docs/added.txt", "docs/common.txt"}) { + t.Fatalf("SnapshotShow: got (%+v, %v)", show, err) + } + aggregate, err := fixture.engine.SnapshotStats(context.Background(), engine.SnapshotStatsRequest{}) + if err != nil || aggregate.SnapshotCount != 3 || aggregate.SnapshotFileCount != 4 { + t.Fatalf("SnapshotStats aggregate: got (%+v, %v)", aggregate, err) + } + perSnapshot, err := fixture.engine.SnapshotStats(context.Background(), engine.SnapshotStatsRequest{SnapshotID: "snap-target"}) + if err != nil || !perSnapshot.HasReuse || perSnapshot.ParentSnapshotID != "snap-base" || perSnapshot.Reused != 1 || perSnapshot.New != 1 { + t.Fatalf("SnapshotStats target: got (%+v, %v)", perSnapshot, err) + } + detailed, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target"}) + if err != nil || detailed.SummaryMode || !reflect.DeepEqual(diffPaths(detailed), []string{"docs/added.txt", "docs/removed.txt"}) { + t.Fatalf("SnapshotDiff detailed: got (%+v, %v)", detailed, err) + } + summary, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target", Summary: true}) + if err != nil || !summary.SummaryMode || summary.Entries != nil || summary.Summary.Added != 1 || summary.Summary.Removed != 1 || summary.MatchedEntryCount != 2 { + t.Fatalf("SnapshotDiff summary: got (%+v, %v)", summary, err) + } + assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) + }) +} + +func TestEngineReadVerifyAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newEngineReadFixture(t, backend) + before := captureEngineReadState(t, backend.DB, fixture.containerDir) + for _, level := range []string{"fast", "standard", "full", "deep"} { + if _, err := fixture.engine.Verify(context.Background(), engine.VerifyRequest{Target: "system", Level: level}); err != nil { + t.Fatalf("Verify system %s: %v", level, err) + } + } + assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) + + if _, err := backend.DB.ExecContext(context.Background(), `INSERT INTO logical_file (original_name, total_size, file_hash, ref_count, status) VALUES ($1, $2, $3, $4, $5)`, "phase7-invalid.txt", 1, "phase7-invalid-hash", 0, "COMPLETED"); err != nil { + t.Fatalf("seed verification inconsistency: %v", err) + } + _, err := fixture.engine.Verify(context.Background(), engine.VerifyRequest{Target: "system", Level: "standard"}) + if err == nil || !strings.Contains(err.Error(), "system standard verification failed") || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + t.Fatalf("Verify inconsistency classification: %v", err) + } + }) +} + +func TestEngineReadContextAndErrorsAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newEngineReadFixture(t, backend) + before := captureEngineReadState(t, backend.DB, fixture.containerDir) + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + operations := []struct { + name string + call func(context.Context) error + }{ + {"Stats", func(ctx context.Context) error { + _, err := fixture.engine.Stats(ctx, engine.StatsRequest{}) + return err + }}, + {"Inspect", func(ctx context.Context) error { + _, err := fixture.engine.Inspect(ctx, engine.InspectRequest{Entity: observability.EntityRepository}) + return err + }}, + {"Verify", func(ctx context.Context) error { + _, err := fixture.engine.Verify(ctx, engine.VerifyRequest{Target: "system", Level: "fast"}) + return err + }}, + {"SnapshotList", func(ctx context.Context) error { + _, err := fixture.engine.SnapshotList(ctx, engine.SnapshotListRequest{}) + return err + }}, + {"SnapshotShow", func(ctx context.Context) error { + _, err := fixture.engine.SnapshotShow(ctx, engine.SnapshotShowRequest{SnapshotID: "snap-target"}) + return err + }}, + {"SnapshotStats", func(ctx context.Context) error { + _, err := fixture.engine.SnapshotStats(ctx, engine.SnapshotStatsRequest{SnapshotID: "snap-target"}) + return err + }}, + {"SnapshotDiff", func(ctx context.Context) error { + _, err := fixture.engine.SnapshotDiff(ctx, engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target"}) + return err + }}, + } + for _, operation := range operations { + t.Run(operation.name, func(t *testing.T) { + err := operation.call(cancelled) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context cancellation, got %v", err) + } + }) + } + if _, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "missing"}); err == nil || !strings.Contains(err.Error(), "not found") || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + t.Fatalf("missing snapshot classification: %v", err) + } + if _, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: "unknown", EntityID: "1"}); err == nil || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + t.Fatalf("invalid inspect classification: %v", err) + } + if _, err := fixture.engine.Verify(context.Background(), engine.VerifyRequest{Target: "unknown"}); err == nil || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + t.Fatalf("invalid verify classification: %v", err) + } + assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) + }) +} + +func equivalentStats(first, second *observability.StatsResult) bool { + if first == nil || second == nil { + return first == second + } + a, b := *first, *second + a.GeneratedAtUTC = time.Time{} + b.GeneratedAtUTC = time.Time{} + return reflect.DeepEqual(a, b) +} + +func assertInspectSummary(t *testing.T, fixture engineReadFixture, entity observability.EntityType, id, key string, want any) { + t.Helper() + result, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: entity, EntityID: id}) + if err != nil || result.Raw == nil || !reflect.DeepEqual(result.Raw.Summary[key], want) { + t.Fatalf("Inspect %s/%s: got (%+v, %v), want summary[%q]=%v", entity, id, result.Raw, err, key, want) + } +} + +func relationsSorted(relations []observability.Relation) bool { + for i := 1; i < len(relations); i++ { + left := fmt.Sprintf("%s|%s|%s|%s", relations[i-1].Direction, relations[i-1].Type, relations[i-1].TargetType, relations[i-1].TargetID) + right := fmt.Sprintf("%s|%s|%s|%s", relations[i].Direction, relations[i].Type, relations[i].TargetType, relations[i].TargetID) + if left > right { + return false + } + } + return true +} + +func snapshotIDs(result engine.SnapshotListResult) []string { + ids := make([]string, len(result.Snapshots)) + for i, snapshot := range result.Snapshots { + ids[i] = snapshot.ID + } + return ids +} + +func snapshotPaths(result engine.SnapshotShowResult) []string { + paths := make([]string, len(result.Files)) + for i, file := range result.Files { + paths[i] = file.StoredPath + } + return paths +} + +func diffPaths(result engine.SnapshotDiffResult) []string { + paths := make([]string, len(result.Entries)) + for i, entry := range result.Entries { + paths[i] = entry.StoredPath + } + return paths +} diff --git a/internal/engine/snapshot_selector_backend_contract_test.go b/internal/engine/snapshot_selector_backend_contract_test.go new file mode 100644 index 00000000..5f0a1d32 --- /dev/null +++ b/internal/engine/snapshot_selector_backend_contract_test.go @@ -0,0 +1,109 @@ +package engine_test + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineSnapshotSelectorsAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newEngineReadFixture(t, backend) + before := captureEngineReadState(t, backend.DB, fixture.containerDir) + + atTie := engineReadFixtureTime.Add(time.Minute) + listed, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{ + Type: engine.SnapshotTypeFull, + Since: &atTie, + Until: &atTie, + Limit: 2, + }) + if err != nil || !reflect.DeepEqual(snapshotIDs(listed), []string{"snap-target", "snap-base"}) { + t.Fatalf("equal-time SnapshotList: got (%+v, %v)", listed, err) + } + filtered, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{Label: "target"}) + if err != nil || !reflect.DeepEqual(snapshotIDs(filtered), []string{"snap-target"}) { + t.Fatalf("label SnapshotList: got (%+v, %v)", filtered, err) + } + + query := engine.SnapshotQuery{ + Path: "docs/added.txt", + Prefix: "docs/", + Pattern: "docs/*.txt", + Regex: "added\\.txt$", + MinSize: int64Pointer(10), + MaxSize: int64Pointer(10), + ModifiedAfter: timePointer(engineReadFixtureTime), + ModifiedBefore: timePointer(engineReadFixtureTime), + } + show, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "snap-target", Query: query}) + if err != nil || !reflect.DeepEqual(snapshotPaths(show), []string{"docs/added.txt"}) || show.MatchedFileCount != 1 || show.TotalFileCount != 2 { + t.Fatalf("filtered SnapshotShow: got (%+v, %v)", show, err) + } + diff, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target", Query: query}) + if err != nil || !reflect.DeepEqual(diffPaths(diff), []string{"docs/added.txt"}) || diff.Summary.Added != 1 || diff.Summary.Removed != 0 { + t.Fatalf("filtered SnapshotDiff: got (%+v, %v)", diff, err) + } + unfilteredShow, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "snap-target"}) + if err != nil || !reflect.DeepEqual(snapshotPaths(unfilteredShow), []string{"docs/added.txt", "docs/common.txt"}) { + t.Fatalf("ordered SnapshotShow: got (%+v, %v)", unfilteredShow, err) + } + unfilteredDiff, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target"}) + if err != nil || !reflect.DeepEqual(diffPaths(unfilteredDiff), []string{"docs/added.txt", "docs/removed.txt"}) { + t.Fatalf("ordered SnapshotDiff: got (%+v, %v)", unfilteredDiff, err) + } + for i := 0; i < 2; i++ { + again, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{Since: &atTie, Until: &atTie}) + if err != nil || !reflect.DeepEqual(snapshotIDs(again), []string{"snap-target", "snap-base"}) { + t.Fatalf("repeated SnapshotList %d: got (%+v, %v)", i, again, err) + } + } + assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) + }) +} + +func TestEngineSnapshotSelectorErrorsAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newEngineReadFixture(t, backend) + before := captureEngineReadState(t, backend.DB, fixture.containerDir) + invalid := engine.SnapshotQuery{Regex: "("} + if _, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "snap-target", Query: invalid}); err == nil || !strings.Contains(err.Error(), "invalid snapshot query regex") { + t.Fatalf("invalid SnapshotShow regex: %v", err) + } + if _, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target", Query: invalid}); err == nil || !strings.Contains(err.Error(), "invalid snapshot query regex") { + t.Fatalf("invalid SnapshotDiff regex: %v", err) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := fixture.engine.SnapshotList(cancelled, engine.SnapshotListRequest{}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled SnapshotList: %v", err) + } + if _, err := fixture.engine.SnapshotShow(cancelled, engine.SnapshotShowRequest{SnapshotID: "snap-target"}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled SnapshotShow: %v", err) + } + if _, err := fixture.engine.SnapshotDiff(cancelled, engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target"}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled SnapshotDiff: %v", err) + } + if _, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "missing"}); err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("missing SnapshotShow: %v", err) + } + if _, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "", TargetID: "snap-target"}); err == nil || !strings.Contains(err.Error(), "base snapshot id cannot be empty") { + t.Fatalf("blank SnapshotDiff base: %v", err) + } + same, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-base"}) + if err != nil || len(same.Entries) != 0 || same.Summary != (engine.SnapshotDiffSummary{}) { + t.Fatalf("same-ID SnapshotDiff: got (%+v, %v)", same, err) + } + assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) + }) +} + +func int64Pointer(value int64) *int64 { return &value } + +func timePointer(value time.Time) *time.Time { return &value } diff --git a/internal/maintenance/gc.go b/internal/maintenance/gc.go index a1a113a7..803715a2 100644 --- a/internal/maintenance/gc.go +++ b/internal/maintenance/gc.go @@ -3,6 +3,8 @@ package maintenance import ( "context" "database/sql" + "database/sql/driver" + "errors" "fmt" "log" "strings" @@ -30,9 +32,10 @@ func isContainerFKViolation(err error) bool { var gcConnectDB = db.ConnectDB -var gcAdvisoryUnlock = func(ctx context.Context, dbconn *sql.DB) error { - _, err := dbconn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", gcAdvisoryLockID) - return err +var gcAdvisoryUnlock = func(ctx context.Context, conn *sql.Conn) (bool, error) { + var unlocked bool + err := conn.QueryRowContext(ctx, "SELECT pg_advisory_unlock($1)", gcAdvisoryLockID).Scan(&unlocked) + return unlocked, err } var gcPhysicalIntegrityCheck = func(dbconn *sql.DB) (verify.PhysicalFileIntegritySummary, error) { @@ -122,11 +125,13 @@ func RunGCWithDB(ctx context.Context, dbconn *sql.DB, dryRun bool, containersDir fsys := fsx.Default() - unlock, err := acquireGCAdvisoryLock(ctx, dbconn, dryRun) + advisoryLock, err := acquireGCAdvisoryLock(ctx, dbconn, dryRun) if err != nil { return GCResult{}, err } - defer unlock() + defer func() { + err = errors.Join(err, advisoryLock.release()) + }() if err := gcIntegrityPreFlight(dbconn); err != nil { return GCResult{}, err @@ -169,31 +174,90 @@ const ( sealedContainerAffected // deleted (or dry-run counted) ) +// gcAdvisoryLock owns the dedicated PostgreSQL session carrying GC's +// session-level advisory lock. The connection must not return to the pool until +// a successful unlock, or until it has been discarded after uncertain cleanup. +type gcAdvisoryLock struct { + conn *sql.Conn +} + // acquireGCAdvisoryLock enforces the SQLite/PostgreSQL backend rules and, for -// PostgreSQL, acquires the advisory lock. Returns an unlock func to defer. -func acquireGCAdvisoryLock(ctx context.Context, dbconn *sql.DB, dryRun bool) (func(), error) { +// PostgreSQL, acquires the advisory lock on one dedicated session. +func acquireGCAdvisoryLock(ctx context.Context, dbconn *sql.DB, dryRun bool) (*gcAdvisoryLock, error) { backend := db.BackendFromDB(dbconn) if backend == db.BackendSQLite { if !dryRun { - return func() {}, fmt.Errorf("live GC is not supported on the SQLite backend; run with --dry-run to inspect GC candidates") + return nil, fmt.Errorf("live GC is not supported on the SQLite backend; run with --dry-run to inspect GC candidates") } log.Println("gc: SQLite backend detected — skipping advisory lock (dry-run only)") - return func() {}, nil + return &gcAdvisoryLock{}, nil } + if backend == db.BackendPostgres && dbconn.Stats().MaxOpenConnections == 1 { + return nil, fmt.Errorf("PostgreSQL GC requires at least two database connections so its dedicated advisory-lock session does not exhaust the pool") + } + + conn, err := dbconn.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("failed to reserve advisory-lock session: %w", err) + } + var locked bool - if err := dbconn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", gcAdvisoryLockID).Scan(&locked); err != nil { - return func() {}, fmt.Errorf("failed to attempt advisory lock: %w", err) + if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", gcAdvisoryLockID).Scan(&locked); err != nil { + acquireErr := fmt.Errorf("failed to attempt advisory lock: %w", err) + return nil, errors.Join(acquireErr, discardGCAdvisoryConn(conn)) } if !locked { - return func() {}, fmt.Errorf("GC already running (advisory lock held)") + return nil, errors.Join( + fmt.Errorf("GC already running (advisory lock held)"), + closeGCAdvisoryConn(conn), + ) } - return func() { - cleanupCtx, cleanupCancel := db.NewOperationContext(context.Background()) - defer cleanupCancel() - if unlockErr := gcAdvisoryUnlock(cleanupCtx, dbconn); unlockErr != nil { - log.Printf("warning: failed to release advisory lock: %v\n", unlockErr) - } - }, nil + return &gcAdvisoryLock{conn: conn}, nil +} + +func (lock *gcAdvisoryLock) release() error { + if lock == nil || lock.conn == nil { + return nil + } + conn := lock.conn + lock.conn = nil + + cleanupCtx, cleanupCancel := db.NewOperationContext(context.Background()) + defer cleanupCancel() + unlocked, unlockErr := gcAdvisoryUnlock(cleanupCtx, conn) + if unlockErr == nil && unlocked { + return closeGCAdvisoryConn(conn) + } + if unlockErr == nil { + unlockErr = fmt.Errorf("PostgreSQL advisory unlock returned false for the owned GC lock") + } else { + unlockErr = fmt.Errorf("failed to release PostgreSQL GC advisory lock: %w", unlockErr) + } + return errors.Join(unlockErr, discardGCAdvisoryConn(conn)) +} + +func closeGCAdvisoryConn(conn *sql.Conn) error { + if conn == nil { + return nil + } + if err := conn.Close(); err != nil && !errors.Is(err, sql.ErrConnDone) { + return fmt.Errorf("close PostgreSQL GC advisory-lock session: %w", err) + } + return nil +} + +// discardGCAdvisoryConn prevents a session whose lock state is uncertain from +// returning to database/sql's reusable pool. driver.ErrBadConn is the expected +// signal used by Conn.Raw to make that connection unusable. +func discardGCAdvisoryConn(conn *sql.Conn) error { + if conn == nil { + return nil + } + err := conn.Raw(func(any) error { return driver.ErrBadConn }) + if err != nil && !errors.Is(err, driver.ErrBadConn) && !errors.Is(err, sql.ErrConnDone) { + return fmt.Errorf("discard PostgreSQL GC advisory-lock session: %w", err) + } + return nil } // gcIntegrityPreFlight runs CheckPhysicalFileGraphIntegrity and returns an @@ -472,7 +536,7 @@ func commitGCContainerDeletion(ctx context.Context, tx *sql.Tx, containerID int6 _ = tx.Rollback() return err } - _, err := tx.ExecContext(ctx, `DELETE FROM container WHERE id = $1`, containerID) + result, err := tx.ExecContext(ctx, `DELETE FROM container WHERE id = $1`, containerID) if err != nil { _ = tx.Rollback() if isContainerFKViolation(err) { @@ -484,6 +548,10 @@ func commitGCContainerDeletion(ctx context.Context, tx *sql.Tx, containerID int6 } return err } + if err := db.RequireExactlyOneRow(result, "delete GC container"); err != nil { + _ = tx.Rollback() + return err + } if err := tx.Commit(); err != nil { return err } @@ -794,10 +862,14 @@ func deletePackedBlockMetadata(ctx context.Context, execer gcSweepExecer, blockI return err } - if _, err := execer.ExecContext(ctx, ` + result, err := execer.ExecContext(ctx, ` DELETE FROM storage_blocks WHERE id = $1 - `, blockID); err != nil { + `, blockID) + if err != nil { + return err + } + if err := db.RequireExactlyOneRow(result, "delete GC packed storage block"); err != nil { return err } @@ -928,7 +1000,12 @@ func sweepDeadActiveContainer(ctx context.Context, dbconn *sql.DB, containersDir } // Delete the container row. - if _, err := tx.ExecContext(ctx, `DELETE FROM container WHERE id = $1`, containerID); err != nil { + result, err := tx.ExecContext(ctx, `DELETE FROM container WHERE id = $1`, containerID) + if err != nil { + _ = tx.Rollback() + return err + } + if err := db.RequireExactlyOneRow(result, "delete fully dead active GC container"); err != nil { _ = tx.Rollback() return err } diff --git a/internal/maintenance/gc_test.go b/internal/maintenance/gc_test.go index d973ece1..42e354bb 100644 --- a/internal/maintenance/gc_test.go +++ b/internal/maintenance/gc_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strings" "testing" + "time" dbschema "github.com/franchoy/coldkeep/db" "github.com/franchoy/coldkeep/internal/blocks" @@ -131,15 +132,23 @@ func setupAdvisoryLockHeldGCFixture(t *testing.T) (*sql.DB, *sql.DB, string, str func holdGCAdvisoryLock(t *testing.T, lockerDB *sql.DB) { t.Helper() + conn, err := lockerDB.Conn(context.Background()) + if err != nil { + t.Fatalf("reserve advisory-lock holder session: %v", err) + } var locked bool - if err := lockerDB.QueryRow(`SELECT pg_try_advisory_lock($1)`, gcAdvisoryLockID).Scan(&locked); err != nil { + if err := conn.QueryRowContext(context.Background(), `SELECT pg_try_advisory_lock($1)`, gcAdvisoryLockID).Scan(&locked); err != nil { + _ = conn.Close() t.Fatalf("acquire advisory lock: %v", err) } if !locked { + _ = conn.Close() t.Fatal("expected to acquire advisory lock in test setup") } t.Cleanup(func() { - _, _ = lockerDB.Exec(`SELECT pg_advisory_unlock($1)`, gcAdvisoryLockID) + var unlocked bool + _ = conn.QueryRowContext(context.Background(), `SELECT pg_advisory_unlock($1)`, gcAdvisoryLockID).Scan(&unlocked) + _ = conn.Close() }) } @@ -159,7 +168,7 @@ func assertGCRefusalPreservesContainerState(t *testing.T, dbconn *sql.DB, contai } } -func TestRunGCWithAdvisoryUnlockFailureStillSucceeds(t *testing.T) { +func TestGCAdvisoryLockUsesDedicatedSessionAndReleases(t *testing.T) { requireDB(t) dbconn, err := db.ConnectDB() @@ -171,51 +180,237 @@ func TestRunGCWithAdvisoryUnlockFailureStillSucceeds(t *testing.T) { applySchema(t, dbconn) resetDB(t, dbconn) - containersDir := t.TempDir() - originalContainersDir := container.ContainersDir - t.Cleanup(func() { - container.ContainersDir = originalContainersDir - }) - container.ContainersDir = containersDir - - filename := "gc-unlock-failure.bin" - containerPath := filepath.Join(containersDir, filename) - if err := os.WriteFile(containerPath, []byte("gc unlock failure test"), 0o600); err != nil { - t.Fatalf("write container file: %v", err) + observer, err := dbconn.Conn(context.Background()) + if err != nil { + t.Fatalf("reserve advisory observer session: %v", err) } + defer observer.Close() - if _, err := dbconn.Exec( - `INSERT INTO container (filename, current_size, max_size, sealed, quarantine) - VALUES ($1, $2, $3, TRUE, FALSE)`, - filename, - int64(len("gc unlock failure test")), - container.GetContainerMaxSize(), - ); err != nil { - t.Fatalf("insert container row: %v", err) + preflightEntered := make(chan struct{}) + resumePreflight := make(chan struct{}) + originalCheck := gcPhysicalIntegrityCheck + gcPhysicalIntegrityCheck = func(_ *sql.DB) (verify.PhysicalFileIntegritySummary, error) { + close(preflightEntered) + <-resumePreflight + return verify.PhysicalFileIntegritySummary{}, nil } + t.Cleanup(func() { gcPhysicalIntegrityCheck = originalCheck }) originalUnlock := gcAdvisoryUnlock - gcAdvisoryUnlock = func(_ context.Context, _ *sql.DB) error { - return errors.New("forced advisory unlock failure") + unlockPID := make(chan int, 1) + gcAdvisoryUnlock = func(ctx context.Context, conn *sql.Conn) (bool, error) { + var pid int + if err := conn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&pid); err != nil { + return false, err + } + unlockPID <- pid + return originalUnlock(ctx, conn) + } + t.Cleanup(func() { gcAdvisoryUnlock = originalUnlock }) + + type gcResult struct { + result GCResult + err error + } + resultCh := make(chan gcResult, 1) + go func() { + result, runErr := RunGCWithDB(context.Background(), dbconn, false, t.TempDir()) + resultCh <- gcResult{result: result, err: runErr} + }() + + select { + case <-preflightEntered: + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for GC preflight after advisory acquisition") + } + + var holderPID int + if err := observer.QueryRowContext(context.Background(), ` + SELECT pid + FROM pg_locks + WHERE locktype = 'advisory' + AND granted + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND classid::bigint = 0 + AND objid::bigint = $1 + AND objsubid = 1 + `, gcAdvisoryLockID).Scan(&holderPID); err != nil { + t.Fatalf("observe GC advisory-lock holder PID: %v", err) + } + if locked := tryGCAdvisoryLockOnConn(t, observer); locked { + t.Fatal("independent session unexpectedly acquired held GC advisory lock") + } + + close(resumePreflight) + select { + case run := <-resultCh: + if run.err != nil { + t.Fatalf("RunGCWithDB: %v", run.err) + } + if run.result.DryRun { + t.Fatalf("expected live GC result, got %+v", run.result) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for GC completion") } - t.Cleanup(func() { - gcAdvisoryUnlock = originalUnlock - }) - result, err := RunGCWithContainersDirResult(false, containersDir) + select { + case pid := <-unlockPID: + if pid != holderPID { + t.Fatalf("advisory unlock PID=%d want acquisition PID=%d", pid, holderPID) + } + default: + t.Fatal("advisory unlock did not report its backend PID") + } + if locked := tryGCAdvisoryLockOnConn(t, observer); !locked { + t.Fatal("independent session could not reacquire advisory lock immediately after GC") + } + unlockGCAdvisoryLockOnConn(t, observer) +} + +func TestRunGCReleasesAdvisoryLockAfterOperationFailure(t *testing.T) { + requireDB(t) + + dbconn, err := db.ConnectDB() + if err != nil { + t.Fatalf("connect db: %v", err) + } + defer dbconn.Close() + applySchema(t, dbconn) + resetDB(t, dbconn) + + observer, err := dbconn.Conn(context.Background()) if err != nil { - t.Fatalf("gc should succeed despite advisory unlock failure: %v", err) + t.Fatalf("reserve advisory observer session: %v", err) } - if result.AffectedContainers != 1 { - t.Fatalf("expected one affected container, got %d", result.AffectedContainers) + defer observer.Close() + + operationErr := errors.New("forced GC operation failure") + originalCheck := gcPhysicalIntegrityCheck + gcPhysicalIntegrityCheck = func(_ *sql.DB) (verify.PhysicalFileIntegritySummary, error) { + return verify.PhysicalFileIntegritySummary{}, operationErr } + t.Cleanup(func() { gcPhysicalIntegrityCheck = originalCheck }) - var remaining int - if err := dbconn.QueryRow(`SELECT COUNT(*) FROM container`).Scan(&remaining); err != nil { - t.Fatalf("count container rows: %v", err) + _, runErr := RunGCWithDB(context.Background(), dbconn, false, t.TempDir()) + if !errors.Is(runErr, operationErr) { + t.Fatalf("RunGCWithDB error=%v want operation failure", runErr) + } + if locked := tryGCAdvisoryLockOnConn(t, observer); !locked { + t.Fatal("independent session could not reacquire advisory lock after GC operation failure") + } + unlockGCAdvisoryLockOnConn(t, observer) +} + +func TestRunGCAdvisoryCleanupFailureReturnsErrorAndDiscardsSession(t *testing.T) { + requireDB(t) + + tests := []struct { + name string + unlockErr error + wantText string + }{ + {name: "SQL error", unlockErr: errors.New("forced advisory unlock failure")}, + {name: "false result", wantText: "returned false"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dbconn, err := db.ConnectDB() + if err != nil { + t.Fatalf("connect db: %v", err) + } + defer dbconn.Close() + applySchema(t, dbconn) + resetDB(t, dbconn) + + observerDB, err := db.ConnectDB() + if err != nil { + t.Fatalf("connect observer db: %v", err) + } + defer observerDB.Close() + observer, err := observerDB.Conn(context.Background()) + if err != nil { + t.Fatalf("reserve observer session: %v", err) + } + defer observer.Close() + + originalUnlock := gcAdvisoryUnlock + var discardedPID int + gcAdvisoryUnlock = func(ctx context.Context, conn *sql.Conn) (bool, error) { + if err := conn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&discardedPID); err != nil { + return false, err + } + return false, test.unlockErr + } + t.Cleanup(func() { gcAdvisoryUnlock = originalUnlock }) + + result, runErr := RunGCWithDB(context.Background(), dbconn, false, t.TempDir()) + if runErr == nil { + t.Fatal("expected advisory cleanup failure") + } + if test.unlockErr != nil && !errors.Is(runErr, test.unlockErr) { + t.Fatalf("cleanup error=%v want errors.Is(%v)", runErr, test.unlockErr) + } + if test.wantText != "" && !strings.Contains(runErr.Error(), test.wantText) { + t.Fatalf("cleanup error=%v want text %q", runErr, test.wantText) + } + if result.DryRun { + t.Fatalf("expected completed live GC result, got %+v", result) + } + if discardedPID == 0 { + t.Fatal("unlock failure did not observe the owning backend PID") + } + + var remaining int + if err := observer.QueryRowContext(context.Background(), ` + SELECT COUNT(*) FROM pg_stat_activity WHERE pid = $1 + `, discardedPID).Scan(&remaining); err != nil { + t.Fatalf("observe discarded advisory session: %v", err) + } + if remaining != 0 { + t.Fatalf("advisory session PID %d remained active after cleanup failure", discardedPID) + } + if locked := tryGCAdvisoryLockOnConn(t, observer); !locked { + t.Fatal("independent session could not acquire advisory lock after failed cleanup discard") + } + unlockGCAdvisoryLockOnConn(t, observer) + }) + } +} + +func TestRunGCLiveRefusesSingleConnectionPool(t *testing.T) { + requireDB(t) + + dbconn, err := db.ConnectDB() + if err != nil { + t.Fatalf("connect db: %v", err) + } + defer dbconn.Close() + dbconn.SetMaxOpenConns(1) + + _, runErr := RunGCWithDB(context.Background(), dbconn, false, t.TempDir()) + if runErr == nil || !strings.Contains(runErr.Error(), "requires at least two database connections") { + t.Fatalf("single-connection GC error=%v want explicit pool-capacity refusal", runErr) + } +} + +func tryGCAdvisoryLockOnConn(t *testing.T, conn *sql.Conn) bool { + t.Helper() + var locked bool + if err := conn.QueryRowContext(context.Background(), `SELECT pg_try_advisory_lock($1)`, gcAdvisoryLockID).Scan(&locked); err != nil { + t.Fatalf("try GC advisory lock: %v", err) + } + return locked +} + +func unlockGCAdvisoryLockOnConn(t *testing.T, conn *sql.Conn) { + t.Helper() + var unlocked bool + if err := conn.QueryRowContext(context.Background(), `SELECT pg_advisory_unlock($1)`, gcAdvisoryLockID).Scan(&unlocked); err != nil { + t.Fatalf("unlock GC advisory lock: %v", err) } - if remaining != 0 { - t.Fatalf("expected container row to be deleted, got %d", remaining) + if !unlocked { + t.Fatal("expected observer session to release GC advisory lock") } } diff --git a/internal/maintenance/mutation_cardinality_test.go b/internal/maintenance/mutation_cardinality_test.go new file mode 100644 index 00000000..41cf26e1 --- /dev/null +++ b/internal/maintenance/mutation_cardinality_test.go @@ -0,0 +1,261 @@ +package maintenance + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/fsx" + _ "github.com/mattn/go-sqlite3" +) + +func openMutationCardinalityMaintenanceDB(t *testing.T) *sql.DB { + t.Helper() + dbconn, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + dbconn.SetMaxOpenConns(1) + t.Cleanup(func() { _ = dbconn.Close() }) + if err := db.RunMigrations(dbconn); err != nil { + t.Fatalf("run migrations: %v", err) + } + return dbconn +} + +func TestRepairRefCountsFailsClosedOnAffectedCountMismatch(t *testing.T) { + t.Run("logical-file", func(t *testing.T) { + dbconn := openMutationCardinalityMaintenanceDB(t) + if _, err := dbconn.Exec(` + INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) + VALUES ('phase17-logical', 1, 'phase17-logical-hash', 'COMPLETED', 1, 'v1-simple-rolling') + `); err != nil { + t.Fatalf("insert logical fixture: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_logical_refcount_repair + BEFORE UPDATE OF ref_count ON logical_file + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create logical repair trigger: %v", err) + } + + _, err := RepairLogicalRefCountsResultWithDB(dbconn) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + var refCount int64 + if err := dbconn.QueryRow(`SELECT ref_count FROM logical_file`).Scan(&refCount); err != nil { + t.Fatalf("read logical refcount: %v", err) + } + if refCount != 1 { + t.Fatalf("logical refcount changed despite rollback: %d", refCount) + } + }) + + t.Run("chunk", func(t *testing.T) { + dbconn := openMutationCardinalityMaintenanceDB(t) + if _, err := dbconn.Exec(` + INSERT INTO chunk (chunk_hash, size, status, live_ref_count, pin_count, chunker_version) + VALUES ('phase17-chunk-hash', 1, 'COMPLETED', 1, 0, 'v1-simple-rolling') + `); err != nil { + t.Fatalf("insert chunk fixture: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_chunk_refcount_repair + BEFORE UPDATE OF live_ref_count ON chunk + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create chunk repair trigger: %v", err) + } + + _, err := RepairChunkLiveRefCountsResultWithDB(dbconn) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + var liveRefCount int64 + if err := dbconn.QueryRow(`SELECT live_ref_count FROM chunk`).Scan(&liveRefCount); err != nil { + t.Fatalf("read chunk refcount: %v", err) + } + if liveRefCount != 1 { + t.Fatalf("chunk refcount changed despite rollback: %d", liveRefCount) + } + }) +} + +func TestGCRequiredDeletesFailClosedOnAffectedCountMismatch(t *testing.T) { + t.Run("sealed-container-keeps-file", func(t *testing.T) { + dbconn := openMutationCardinalityMaintenanceDB(t) + dir := t.TempDir() + filename := "phase17-sealed-gc.bin" + path := filepath.Join(dir, filename) + if err := os.WriteFile(path, []byte("phase17"), 0o600); err != nil { + t.Fatalf("write container fixture: %v", err) + } + result, err := dbconn.Exec( + `INSERT INTO container (filename, current_size, max_size, sealed, quarantine) + VALUES (?, 7, ?, TRUE, FALSE)`, + filename, + container.GetContainerMaxSize(), + ) + if err != nil { + t.Fatalf("insert sealed container: %v", err) + } + containerID, err := result.LastInsertId() + if err != nil { + t.Fatalf("container id: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_sealed_container_delete + BEFORE DELETE ON container + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create sealed-delete trigger: %v", err) + } + + tx, err := dbconn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin GC transaction: %v", err) + } + err = commitGCContainerDeletion(context.Background(), tx, containerID, dir, filename, fsx.Default()) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("physical file removed after missed metadata delete: %v", err) + } + var remaining int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM container WHERE id = ?`, containerID).Scan(&remaining); err != nil { + t.Fatalf("count container rows: %v", err) + } + if remaining != 1 { + t.Fatalf("container rollback count=%d, want 1", remaining) + } + }) + + t.Run("packed-storage-block", func(t *testing.T) { + dbconn := openMutationCardinalityMaintenanceDB(t) + containerResult, err := dbconn.Exec( + `INSERT INTO container (filename, current_size, max_size, sealed, quarantine) + VALUES ('phase17-packed-gc.bin', 0, ?, TRUE, FALSE)`, + container.GetContainerMaxSize(), + ) + if err != nil { + t.Fatalf("insert packed container: %v", err) + } + containerID, err := containerResult.LastInsertId() + if err != nil { + t.Fatalf("container id: %v", err) + } + blockResult, err := dbconn.Exec(` + INSERT INTO storage_blocks + (format_version, codec, plaintext_size, compression_codec, stored_size, container_id, container_offset, block_hash) + VALUES (1, 'none', 1, 'none', 1, ?, 0, X'01') + `, containerID) + if err != nil { + t.Fatalf("insert packed block: %v", err) + } + blockID, err := blockResult.LastInsertId() + if err != nil { + t.Fatalf("block id: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_storage_block_delete + BEFORE DELETE ON storage_blocks + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create block-delete trigger: %v", err) + } + + tx, err := dbconn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin packed-block transaction: %v", err) + } + err = deletePackedBlockMetadata(context.Background(), tx, blockID) + if !errors.Is(err, db.ErrMutationCardinality) { + _ = tx.Rollback() + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback packed-block transaction: %v", err) + } + var remaining int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM storage_blocks WHERE id = ?`, blockID).Scan(&remaining); err != nil { + t.Fatalf("count storage blocks: %v", err) + } + if remaining != 1 { + t.Fatalf("storage block rollback count=%d, want 1", remaining) + } + }) + + t.Run("active-container-keeps-file", func(t *testing.T) { + dbconn := openMutationCardinalityMaintenanceDB(t) + dir := t.TempDir() + filename := "phase17-active-gc.bin" + path := filepath.Join(dir, filename) + if err := os.WriteFile(path, []byte("phase17"), 0o600); err != nil { + t.Fatalf("write active container fixture: %v", err) + } + result, err := dbconn.Exec( + `INSERT INTO container (filename, current_size, max_size, sealed, quarantine) + VALUES (?, 7, ?, FALSE, FALSE)`, + filename, + container.GetContainerMaxSize(), + ) + if err != nil { + t.Fatalf("insert active container: %v", err) + } + containerID, err := result.LastInsertId() + if err != nil { + t.Fatalf("container id: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_active_container_delete + BEFORE DELETE ON container + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create active-delete trigger: %v", err) + } + + err = sweepDeadActiveContainer( + context.Background(), + dbconn, + dir, + livePhysicalUnits{ + LegacyLiveContainerIDs: map[int64]struct{}{}, + PackedLiveBlockIDs: map[int64]struct{}{}, + }, + fsx.Default(), + containerID, + filename, + ) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("active physical file removed after missed metadata delete: %v", err) + } + var remaining int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM container WHERE id = ?`, containerID).Scan(&remaining); err != nil { + t.Fatalf("count active container rows: %v", err) + } + if remaining != 1 { + t.Fatalf("active container rollback count=%d, want 1", remaining) + } + }) +} diff --git a/internal/maintenance/repair_chunk_refcounts.go b/internal/maintenance/repair_chunk_refcounts.go index 59a1d72e..8fbfb3bb 100644 --- a/internal/maintenance/repair_chunk_refcounts.go +++ b/internal/maintenance/repair_chunk_refcounts.go @@ -53,7 +53,7 @@ func RepairChunkLiveRefCountsResultWithDB(dbconn *sql.DB) (result RepairChunkLiv } if result.UpdatedChunks > 0 { - if _, err := tx.ExecContext(ctx, ` + mutationResult, err := tx.ExecContext(ctx, ` UPDATE chunk SET live_ref_count = ( SELECT COUNT(*) @@ -65,7 +65,11 @@ func RepairChunkLiveRefCountsResultWithDB(dbconn *sql.DB) (result RepairChunkLiv FROM file_chunk fc WHERE fc.chunk_id = chunk.id ) - `); err != nil { + `) + if err != nil { + return RepairChunkLiveRefCountsResult{}, fmt.Errorf("update chunk.live_ref_count from file_chunk rows: %w", err) + } + if err := db.RequireRowsAffected(mutationResult, "repair chunk live refcounts", result.UpdatedChunks); err != nil { return RepairChunkLiveRefCountsResult{}, fmt.Errorf("update chunk.live_ref_count from file_chunk rows: %w", err) } } diff --git a/internal/maintenance/repair_refcounts.go b/internal/maintenance/repair_refcounts.go index 66e2aeae..af83d621 100644 --- a/internal/maintenance/repair_refcounts.go +++ b/internal/maintenance/repair_refcounts.go @@ -74,7 +74,7 @@ func RepairLogicalRefCountsResultWithDB(dbconn *sql.DB) (result RepairLogicalRef } if result.UpdatedLogicalFiles > 0 { - if _, err := tx.ExecContext(ctx, ` + mutationResult, err := tx.ExecContext(ctx, ` UPDATE logical_file SET ref_count = ( SELECT COUNT(*) @@ -86,7 +86,11 @@ func RepairLogicalRefCountsResultWithDB(dbconn *sql.DB) (result RepairLogicalRef FROM physical_file pf WHERE pf.logical_file_id = logical_file.id ) - `); err != nil { + `) + if err != nil { + return RepairLogicalRefCountsResult{}, fmt.Errorf("update logical_file.ref_count from physical_file rows: %w", err) + } + if err := db.RequireRowsAffected(mutationResult, "repair logical file refcounts", result.UpdatedLogicalFiles); err != nil { return RepairLogicalRefCountsResult{}, fmt.Errorf("update logical_file.ref_count from physical_file rows: %w", err) } } diff --git a/internal/recovery/mutation_cardinality_test.go b/internal/recovery/mutation_cardinality_test.go new file mode 100644 index 00000000..d506615f --- /dev/null +++ b/internal/recovery/mutation_cardinality_test.go @@ -0,0 +1,63 @@ +package recovery + +import ( + "context" + "database/sql" + "errors" + "testing" + + "github.com/franchoy/coldkeep/internal/db" + _ "github.com/mattn/go-sqlite3" +) + +func TestOrphanResyncFailsClosedWhenUpdateMatchesZero(t *testing.T) { + dbconn, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + dbconn.SetMaxOpenConns(1) + defer func() { _ = dbconn.Close() }() + if err := db.RunMigrations(dbconn); err != nil { + t.Fatalf("run migrations: %v", err) + } + + const filename = "phase17-orphan.bin" + if _, err := dbconn.Exec(` + INSERT INTO container (filename, current_size, max_size, sealed, sealing, quarantine) + VALUES (?, 64, 64, FALSE, FALSE, TRUE) + `, filename); err != nil { + t.Fatalf("insert quarantined orphan row: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_orphan_resync + BEFORE UPDATE OF current_size ON container + WHEN OLD.filename = 'phase17-orphan.bin' + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create ignored-resync trigger: %v", err) + } + + reused, skipped, err := resolveOrphanConflictWithFS( + context.Background(), + dbconn, + db.BackendSQLite, + filename, + 128, + ) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if reused || skipped { + t.Fatalf("mismatch reported success: reused=%t skipped=%t", reused, skipped) + } + + var currentSize, maxSize int64 + if err := dbconn.QueryRow(`SELECT current_size, max_size FROM container WHERE filename = ?`, filename).Scan(¤tSize, &maxSize); err != nil { + t.Fatalf("read orphan row after failed resync: %v", err) + } + if currentSize != 64 || maxSize != 64 { + t.Fatalf("orphan sizes changed: current=%d max=%d", currentSize, maxSize) + } +} diff --git a/internal/recovery/system_recovery.go b/internal/recovery/system_recovery.go index 6f8a611a..82e6f12a 100644 --- a/internal/recovery/system_recovery.go +++ b/internal/recovery/system_recovery.go @@ -237,7 +237,7 @@ func recoverOneSealingContainer(ctx context.Context, dbconn *sql.DB, id int64, f fileInfo, statErr := fsys.Stat(path) if statErr == nil && fileInfo.Size() != currentSize { if _, qErr := dbconn.ExecContext(ctx, - `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = $2, max_size = $2 WHERE id = $1`, + `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = $2 WHERE id = $1`, id, fileInfo.Size(), ); qErr != nil { @@ -483,7 +483,7 @@ func checkActiveContainerIntegrity(ctx context.Context, dbconn *sql.DB, id int64 // quarantineOneActiveCorruptTail marks a container as quarantined in the DB // and logs the corrective-recovery event. func quarantineOneActiveCorruptTail(ctx context.Context, dbconn *sql.DB, id int64, filename string, currentSize, physicalSize int64, reason string, stats *recoveryStats) error { - _, err := dbconn.ExecContext(ctx, `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = $2, max_size = $2 WHERE id = $1`, id, physicalSize) + _, err := dbconn.ExecContext(ctx, `UPDATE container SET quarantine = TRUE, sealing = FALSE, current_size = $2 WHERE id = $1`, id, physicalSize) if err != nil { return fmt.Errorf("query update active container to quarantine due to corrupt tail: %w", err) } @@ -609,13 +609,29 @@ func resolveOrphanConflictWithFS(ctx context.Context, dbconn *sql.DB, backend db } if existingQuarantine { - resyncQuery := `UPDATE container SET current_size = $2, max_size = $2 WHERE filename = $1` + resyncQuery := `UPDATE container SET current_size = $2 WHERE filename = $1` resyncArgs := []any{name, fileSize} if backend == db.BackendSQLite { - resyncQuery = `UPDATE container SET current_size = ?, max_size = ? WHERE filename = ?` - resyncArgs = []any{fileSize, fileSize, name} + resyncQuery = `UPDATE container SET current_size = ? WHERE filename = ?` + resyncArgs = []any{fileSize, name} } - if _, err := dbconn.ExecContext(ctx, resyncQuery, resyncArgs...); err != nil { + // Orphan quarantine rows use max_size as a physical-size marker because + // there is no trusted container header. Keep that marker synchronized + // when both stored sizes still identify the same orphan artifact. A real + // container row has a distinct persisted capacity and must retain it. + if existingMaxSize == existingCurrentSize { + resyncQuery = `UPDATE container SET current_size = $2, max_size = $2 WHERE filename = $1` + resyncArgs = []any{name, fileSize} + if backend == db.BackendSQLite { + resyncQuery = `UPDATE container SET current_size = ?, max_size = ? WHERE filename = ?` + resyncArgs = []any{fileSize, fileSize, name} + } + } + result, err := dbconn.ExecContext(ctx, resyncQuery, resyncArgs...) + if err != nil { + return false, false, fmt.Errorf("resync quarantined orphan container %s: %w", name, err) + } + if err := db.RequireExactlyOneRow(result, "resync quarantined orphan container"); err != nil { return false, false, fmt.Errorf("resync quarantined orphan container %s: %w", name, err) } logRecoveryEvent( diff --git a/internal/recovery/system_recovery_test.go b/internal/recovery/system_recovery_test.go index 04d2033c..da142254 100644 --- a/internal/recovery/system_recovery_test.go +++ b/internal/recovery/system_recovery_test.go @@ -12,7 +12,7 @@ import ( _ "github.com/mattn/go-sqlite3" ) -func TestQuarantineOrphanContainersAcceptsDirectQuarantineAfterSizeSync(t *testing.T) { +func TestQuarantineOrphanContainersAcceptsDirectQuarantineAfterCurrentSizeSync(t *testing.T) { dbconn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("open sqlite db: %v", err) @@ -74,8 +74,8 @@ func TestQuarantineOrphanContainersAcceptsDirectQuarantineAfterSizeSync(t *testi if err != nil { t.Fatalf("stat container file: %v", err) } - if currentSize != info.Size() || maxSize != info.Size() { - t.Fatalf("expected direct quarantine size sync to %d, got current=%d max=%d", info.Size(), currentSize, maxSize) + if currentSize != info.Size() || maxSize != container.ContainerHdrLen+256 { + t.Fatalf("expected current size sync to %d and preserved maximum %d, got current=%d max=%d", info.Size(), container.ContainerHdrLen+256, currentSize, maxSize) } stats := &recoveryStats{} @@ -87,7 +87,7 @@ func TestQuarantineOrphanContainersAcceptsDirectQuarantineAfterSizeSync(t *testi } } -func TestQuarantineOrphanContainersResyncsExistingQuarantineSizeDrift(t *testing.T) { +func TestQuarantineOrphanContainersResyncsCurrentSizeAndPreservesMaximum(t *testing.T) { dbconn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("open sqlite db: %v", err) @@ -111,7 +111,7 @@ func TestQuarantineOrphanContainersResyncsExistingQuarantineSizeDrift(t *testing res, err := dbconn.Exec(` INSERT INTO container (filename, current_size, max_size, sealed, sealing, quarantine) VALUES (?, ?, ?, FALSE, FALSE, TRUE) - `, filename, int64(len(content))-5, int64(len(content))-5) + `, filename, int64(len(content))-5, int64(len(content))+64) if err != nil { t.Fatalf("insert stale quarantine row: %v", err) } @@ -134,8 +134,8 @@ func TestQuarantineOrphanContainersResyncsExistingQuarantineSizeDrift(t *testing if quarantine != 1 { t.Fatalf("expected row to remain quarantined, got %d", quarantine) } - if currentSize != int64(len(content)) || maxSize != int64(len(content)) { - t.Fatalf("expected resynced sizes=%d, got current=%d max=%d", len(content), currentSize, maxSize) + if currentSize != int64(len(content)) || maxSize != int64(len(content))+64 { + t.Fatalf("expected current size=%d and preserved maximum=%d, got current=%d max=%d", len(content), len(content)+64, currentSize, maxSize) } if stats.quarantinedOrphan != 0 { t.Fatalf("expected no new orphan rows, got %d", stats.quarantinedOrphan) diff --git a/internal/storage/compression/compression.go b/internal/storage/compression/compression.go index 93bd930f..d74b47d2 100644 --- a/internal/storage/compression/compression.go +++ b/internal/storage/compression/compression.go @@ -9,6 +9,12 @@ const ( CompressionNone = "none" CompressionZstd = "zstd" + // MaxDecompressedBlockSize is the maximum complete encoded CKBL block + // accepted by the storage decompression boundary. Released writers produce + // at most a 3 MiB chunk payload plus CKBL header/table overhead, which is + // strictly below this 4 MiB format/runtime ceiling. + MaxDecompressedBlockSize int64 = 4 << 20 + // DefaultCompressionCodec keeps compression disabled unless explicitly enabled. DefaultCompressionCodec = CompressionNone DefaultCompressionLevel = 3 @@ -19,12 +25,52 @@ const ( var ErrInvalidCompressionLevel = fmt.Errorf("invalid compression level") // Compressor provides a codec-stable compression/decompression contract. +// Decompress requires a known expected size in [0, MaxDecompressedBlockSize]. +// A successful decode always returns exactly expectedSize bytes. type Compressor interface { Codec() string Compress(input []byte) ([]byte, error) Decompress(input []byte, expectedSize int64) ([]byte, error) } +func validateDecompressionExpectation(codec string, expectedSize, maxOutput int64) error { + if maxOutput <= 0 { + return newCompressionError( + ErrCompressionSizeMismatch, + 0, + codec, + expectedSize, + -1, + fmt.Errorf("invalid decompression maximum: %d", maxOutput), + ) + } + if expectedSize < 0 || expectedSize > maxOutput { + return newCompressionError( + ErrCompressionSizeMismatch, + 0, + codec, + expectedSize, + -1, + fmt.Errorf("expected size outside permitted range [0,%d]", maxOutput), + ) + } + return nil +} + +func validateDecompressedSize(codec string, actualSize, expectedSize int64) error { + if actualSize == expectedSize { + return nil + } + return newCompressionError( + ErrCompressionSizeMismatch, + 0, + codec, + expectedSize, + actualSize, + nil, + ) +} + // Lookup returns a compressor implementation for the requested codec. // Empty codec maps to "none" for compatibility with legacy metadata defaults. func Lookup(codec string) (Compressor, error) { diff --git a/internal/storage/compression/compression_test.go b/internal/storage/compression/compression_test.go index 8024f0ce..eca21aee 100644 --- a/internal/storage/compression/compression_test.go +++ b/internal/storage/compression/compression_test.go @@ -77,6 +77,37 @@ func TestNoneRoundTripUnchanged(t *testing.T) { } } +func TestNoneDecompressRequiresExactExpectedSize(t *testing.T) { + compressor, err := Lookup(CompressionNone) + if err != nil { + t.Fatalf("Lookup none: %v", err) + } + + input := []byte("identity-no-copy") + recovered, err := compressor.Decompress(input, int64(len(input))) + if err != nil { + t.Fatalf("exact none decompress: %v", err) + } + if len(input) > 0 && &recovered[0] != &input[0] { + t.Fatal("none decompression copied the input") + } + + for _, expectedSize := range []int64{-1, int64(len(input) - 1), int64(len(input) + 1), MaxDecompressedBlockSize + 1} { + _, err := compressor.Decompress(input, expectedSize) + if !errors.Is(err, ErrCompressionSizeMismatch) { + t.Fatalf("expected size mismatch for expected=%d, got: %v", expectedSize, err) + } + } + + empty, err := compressor.Decompress([]byte{}, 0) + if err != nil { + t.Fatalf("empty none decompress: %v", err) + } + if len(empty) != 0 { + t.Fatalf("empty none output length: got=%d", len(empty)) + } +} + func TestZstdRoundTripExact(t *testing.T) { compressor, err := Lookup(CompressionZstd) if err != nil { diff --git a/internal/storage/compression/none.go b/internal/storage/compression/none.go index 7f525c2d..ffda3b4e 100644 --- a/internal/storage/compression/none.go +++ b/internal/storage/compression/none.go @@ -10,6 +10,12 @@ func (noneCompressor) Compress(input []byte) ([]byte, error) { return input, nil } -func (noneCompressor) Decompress(input []byte, _ int64) ([]byte, error) { +func (noneCompressor) Decompress(input []byte, expectedSize int64) ([]byte, error) { + if err := validateDecompressionExpectation(CompressionNone, expectedSize, MaxDecompressedBlockSize); err != nil { + return nil, err + } + if err := validateDecompressedSize(CompressionNone, int64(len(input)), expectedSize); err != nil { + return nil, err + } return input, nil } diff --git a/internal/storage/compression/zstd.go b/internal/storage/compression/zstd.go index 21e095b7..2f886370 100644 --- a/internal/storage/compression/zstd.go +++ b/internal/storage/compression/zstd.go @@ -1,7 +1,9 @@ package compression import ( + "errors" "fmt" + "strings" "github.com/klauspost/compress/zstd" ) @@ -42,20 +44,45 @@ func (c zstdCompressor) Compress(input []byte) ([]byte, error) { } func (zstdCompressor) Decompress(input []byte, expectedSize int64) ([]byte, error) { - decoder, err := zstd.NewReader(nil) + return decompressZstdBounded(input, expectedSize, MaxDecompressedBlockSize) +} + +func decompressZstdBounded(input []byte, expectedSize, maxOutput int64) ([]byte, error) { + if err := validateDecompressionExpectation(CompressionZstd, expectedSize, maxOutput); err != nil { + return nil, err + } + + decoder, err := zstd.NewReader( + nil, + zstd.WithDecoderMaxMemory(uint64(maxOutput)), + zstd.WithDecodeAllCapLimit(true), + ) if err != nil { return nil, newCompressionError(ErrDecompressionFailed, 0, CompressionZstd, expectedSize, -1, err) } defer decoder.Close() - output, err := decoder.DecodeAll(input, nil) + output, err := decoder.DecodeAll(input, make([]byte, 0, int(expectedSize))) if err != nil { + if isZstdDecompressionLimitError(err) { + return nil, newCompressionError(ErrCompressionSizeMismatch, 0, CompressionZstd, expectedSize, -1, err) + } return nil, newCompressionError(ErrDecompressionFailed, 0, CompressionZstd, expectedSize, -1, fmt.Errorf("invalid compressed input: %w", err)) } - if expectedSize >= 0 && int64(len(output)) != expectedSize { - return nil, newCompressionError(ErrCompressionSizeMismatch, 0, CompressionZstd, expectedSize, int64(len(output)), nil) + if err := validateDecompressedSize(CompressionZstd, int64(len(output)), expectedSize); err != nil { + return nil, err } return output, nil } + +func isZstdDecompressionLimitError(err error) bool { + return errors.Is(err, zstd.ErrDecoderSizeExceeded) || + errors.Is(err, zstd.ErrWindowSizeExceeded) || + // klauspost/compress v1.18.0 has SIMD decode paths that report a + // cap-limit breach with this non-sentinel error instead of + // ErrDecoderSizeExceeded. The dependency is pinned, and this keeps all + // of its bounded-output paths under Coldkeep's size-mismatch contract. + strings.Contains(err.Error(), "output bigger than max block size") +} diff --git a/internal/storage/compression/zstd_test.go b/internal/storage/compression/zstd_test.go index ac937112..7d575973 100644 --- a/internal/storage/compression/zstd_test.go +++ b/internal/storage/compression/zstd_test.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "errors" + "math" "math/rand" "strings" "testing" @@ -59,7 +60,7 @@ func TestZstdInvalidCompressedInputReturnsCleanError(t *testing.T) { t.Fatalf("NewZstdCompressor: %v", err) } - _, err = compressor.Decompress([]byte("not-zstd"), -1) + _, err = compressor.Decompress([]byte("not-zstd"), 1) if err == nil { t.Fatal("expected error for invalid zstd payload") } @@ -78,6 +79,88 @@ func TestZstdInvalidCompressedInputReturnsCleanError(t *testing.T) { } } +func TestZstdDecompressRejectsExpectedSizeOutsideAbsoluteBound(t *testing.T) { + tests := []struct { + expectedSize int64 + maxOutput int64 + }{ + {expectedSize: -1, maxOutput: MaxDecompressedBlockSize}, + {expectedSize: MaxDecompressedBlockSize + 1, maxOutput: MaxDecompressedBlockSize}, + {expectedSize: math.MaxInt64, maxOutput: 64}, + } + for _, tc := range tests { + _, err := decompressZstdBounded([]byte("not-zstd"), tc.expectedSize, tc.maxOutput) + if !errors.Is(err, ErrCompressionSizeMismatch) { + t.Fatalf("expected pre-decode size mismatch for expected=%d max=%d, got: %v", tc.expectedSize, tc.maxOutput, err) + } + if errors.Is(err, ErrDecompressionFailed) { + t.Fatalf("decoder ran before expectation validation for expected=%d max=%d: %v", tc.expectedSize, tc.maxOutput, err) + } + } +} + +func TestZstdDecompressRejectsOutputBeyondExpectedSize(t *testing.T) { + compressor, err := NewZstdCompressor(3) + if err != nil { + t.Fatalf("NewZstdCompressor: %v", err) + } + payload := bytes.Repeat([]byte("a"), 1024) + compressed, err := compressor.Compress(payload) + if err != nil { + t.Fatalf("Compress: %v", err) + } + + _, err = decompressZstdBounded(compressed, 64, 64<<10) + if !errors.Is(err, ErrCompressionSizeMismatch) { + t.Fatalf("expected bounded size mismatch, got: %v", err) + } +} + +func TestZstdDecompressRejectsTruncatedInput(t *testing.T) { + compressor, err := NewZstdCompressor(3) + if err != nil { + t.Fatalf("NewZstdCompressor: %v", err) + } + payload := bytes.Repeat([]byte("truncated-zstd"), 32) + compressed, err := compressor.Compress(payload) + if err != nil { + t.Fatalf("Compress: %v", err) + } + compressed = compressed[:len(compressed)-1] + + _, err = compressor.Decompress(compressed, int64(len(payload))) + if !errors.Is(err, ErrDecompressionFailed) { + t.Fatalf("expected ErrDecompressionFailed, got: %v", err) + } +} + +func TestZstdDecompressBoundsConcatenatedFramesAcrossAggregateOutput(t *testing.T) { + compressor, err := NewZstdCompressor(3) + if err != nil { + t.Fatalf("NewZstdCompressor: %v", err) + } + payload := bytes.Repeat([]byte("frame"), 32) + frame, err := compressor.Compress(payload) + if err != nil { + t.Fatalf("Compress: %v", err) + } + concatenated := append(append([]byte(nil), frame...), frame...) + + _, err = decompressZstdBounded(concatenated, int64(len(payload)), 64<<10) + if !errors.Is(err, ErrCompressionSizeMismatch) { + t.Fatalf("expected aggregate output bound failure, got: %v", err) + } + + want := append(append([]byte(nil), payload...), payload...) + got, err := decompressZstdBounded(concatenated, int64(len(want)), 64<<10) + if err != nil { + t.Fatalf("exact concatenated decode: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatal("concatenated decode mismatch") + } +} + func TestZstdDecompressSizeMismatchReturnsTypedError(t *testing.T) { compressor, err := NewZstdCompressor(3) if err != nil { diff --git a/internal/storage/mutation_cardinality_test.go b/internal/storage/mutation_cardinality_test.go new file mode 100644 index 00000000..d6e96467 --- /dev/null +++ b/internal/storage/mutation_cardinality_test.go @@ -0,0 +1,259 @@ +package storage + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/franchoy/coldkeep/internal/blocks" + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/db" + filestate "github.com/franchoy/coldkeep/internal/status" + _ "github.com/mattn/go-sqlite3" +) + +func openMutationCardinalityStorageDB(t *testing.T) *sql.DB { + t.Helper() + dsn := filepath.Join(t.TempDir(), "phase17.sqlite") + "?_foreign_keys=on" + dbconn, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = dbconn.Close() }) + if err := db.RunMigrations(dbconn); err != nil { + t.Fatalf("run migrations: %v", err) + } + return dbconn +} + +func insertMutationCardinalityLogicalFile(t *testing.T, dbconn *sql.DB, name, hash, status string, refCount int64) int64 { + t.Helper() + var fileID int64 + if err := dbconn.QueryRow(` + INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) + VALUES (?, 1, ?, ?, ?, 'v1-simple-rolling') + RETURNING id + `, name, hash, status, refCount).Scan(&fileID); err != nil { + t.Fatalf("insert logical file: %v", err) + } + return fileID +} + +func TestPhysicalFileMutationsFailClosedOnCardinalityMismatch(t *testing.T) { + t.Run("missing-update", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + err := updatePhysicalFile(context.Background(), dbconn, "/phase17/missing", 404, physicalFileMetadata{}) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) + + t.Run("missing-increment", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + err := incrementLogicalFileRefCount(context.Background(), dbconn, 404) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) + + t.Run("zero-refcount-decrement", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + fileID := insertMutationCardinalityLogicalFile(t, dbconn, "zero-ref", "phase17-zero-ref", filestate.LogicalFileCompleted, 0) + err := decrementLogicalFileRefCount(context.Background(), dbconn, fileID) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) + + t.Run("existing-same-value-update", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + fileID := insertMutationCardinalityLogicalFile(t, dbconn, "same-value", "phase17-same-value", filestate.LogicalFileCompleted, 1) + const path = "/phase17/same-value" + meta := physicalFileMetadata{IsMetadataComplete: true} + if _, err := dbconn.Exec(` + INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) + VALUES (?, ?, TRUE) + `, path, fileID); err != nil { + t.Fatalf("insert physical file: %v", err) + } + if err := updatePhysicalFile(context.Background(), dbconn, path, fileID, meta); err != nil { + t.Fatalf("same-value update must match one row: %v", err) + } + }) + + t.Run("ignored-replacement-delete", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + oldFileID := insertMutationCardinalityLogicalFile(t, dbconn, "old", "phase17-old", filestate.LogicalFileCompleted, 1) + newFileID := insertMutationCardinalityLogicalFile(t, dbconn, "new", "phase17-new", filestate.LogicalFileCompleted, 0) + const path = "/phase17/replace" + if _, err := dbconn.Exec(`INSERT INTO physical_file (path, logical_file_id) VALUES (?, ?)`, path, oldFileID); err != nil { + t.Fatalf("insert physical mapping: %v", err) + } + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_physical_replace_delete + BEFORE DELETE ON physical_file + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create ignored-delete trigger: %v", err) + } + + tx, err := dbconn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin replace transaction: %v", err) + } + err = replacePhysicalFileLogicalTargetTx(context.Background(), dbconn, tx, path, newFileID, physicalFileMetadata{}) + if !errors.Is(err, db.ErrMutationCardinality) { + _ = tx.Rollback() + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback replacement: %v", err) + } + + var mappedID int64 + if err := dbconn.QueryRow(`SELECT logical_file_id FROM physical_file WHERE path = ?`, path).Scan(&mappedID); err != nil { + t.Fatalf("read mapping after rollback: %v", err) + } + if mappedID != oldFileID { + t.Fatalf("mapping changed after rollback: got=%d want=%d", mappedID, oldFileID) + } + }) +} + +func TestStoreRequiredMutationCardinalityFailuresRollBack(t *testing.T) { + t.Run("container-sealing", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + tx, err := dbconn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin sealing transaction: %v", err) + } + err = markContainerSealingInTx(tx, 404) + if !errors.Is(err, db.ErrMutationCardinality) { + _ = tx.Rollback() + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + _ = tx.Rollback() + }) + + t.Run("linked-chunk-refcount", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + fileID := insertMutationCardinalityLogicalFile(t, dbconn, "link", "phase17-link", filestate.LogicalFileProcessing, 0) + dbconn.SetMaxOpenConns(1) + dbconn.SetMaxIdleConns(1) + if _, err := dbconn.Exec(`PRAGMA foreign_keys = OFF`); err != nil { + t.Fatalf("disable fixture foreign keys: %v", err) + } + tx, err := dbconn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin link transaction: %v", err) + } + err = linkFileChunkWithContext(context.Background(), tx, fileID, 404, 0, true) + if !errors.Is(err, db.ErrMutationCardinality) { + _ = tx.Rollback() + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback link transaction: %v", err) + } + var mappings int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM file_chunk WHERE logical_file_id = ?`, fileID).Scan(&mappings); err != nil { + t.Fatalf("count rolled-back mappings: %v", err) + } + if mappings != 0 { + t.Fatalf("link mismatch committed %d mappings", mappings) + } + }) + + t.Run("missing-empty-logical-finalization", func(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + err := finalizeLogicalFileStorageWithContext(context.Background(), dbconn, 404, 0) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + }) + + t.Run("new-chunk-completion", func(t *testing.T) { + dbconn, sgctx, path, codec := setupMutationCardinalityStore(t) + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_new_chunk_completion + BEFORE UPDATE OF status ON chunk + WHEN NEW.status = 'COMPLETED' + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create chunk completion trigger: %v", err) + } + + _, err := StoreFileWithStorageContextAndCodecResult(sgctx, path, codec) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + var completed int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM chunk WHERE status = 'COMPLETED'`).Scan(&completed); err != nil { + t.Fatalf("count completed chunks: %v", err) + } + if completed != 0 { + t.Fatalf("completion mismatch committed %d completed chunks", completed) + } + }) + +} + +func setupMutationCardinalityStore(t *testing.T) (*sql.DB, StorageContext, string, blocks.Codec) { + t.Helper() + dbconn := openMutationCardinalityStorageDB(t) + if _, err := dbconn.Exec( + `INSERT INTO container (id, filename, current_size, max_size, sealed, quarantine) + VALUES (1, 'ack_test_container.bin', ?, ?, FALSE, FALSE)`, + container.ContainerHdrLen, + container.GetContainerMaxSize(), + ); err != nil { + t.Fatalf("insert store container: %v", err) + } + dir := t.TempDir() + path := filepath.Join(dir, "phase17-store.txt") + if err := os.WriteFile(path, []byte("phase17 mutation cardinality store fixture"), 0o600); err != nil { + t.Fatalf("write store fixture: %v", err) + } + codec, err := blocks.ParseCodec("plain") + if err != nil { + t.Fatalf("parse plain codec: %v", err) + } + return dbconn, StorageContext{ + DB: dbconn, + Writer: &commitAckWriter{}, + ContainerDir: dir, + }, path, codec +} + +func TestRemoveFileFailsClosedWhenLogicalDeleteMatchesZero(t *testing.T) { + dbconn := openMutationCardinalityStorageDB(t) + fileID := insertMutationCardinalityLogicalFile(t, dbconn, "remove", "phase17-remove", filestate.LogicalFileCompleted, 0) + if _, err := dbconn.Exec(` + CREATE TRIGGER phase17_ignore_logical_file_delete + BEFORE DELETE ON logical_file + BEGIN + SELECT RAISE(IGNORE); + END + `); err != nil { + t.Fatalf("create ignored logical-delete trigger: %v", err) + } + + _, err := RemoveFileWithDBResult(dbconn, fileID) + if !errors.Is(err, db.ErrMutationCardinality) { + t.Fatalf("error=%v, want ErrMutationCardinality", err) + } + var remaining int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM logical_file WHERE id = ?`, fileID).Scan(&remaining); err != nil { + t.Fatalf("count logical files after rollback: %v", err) + } + if remaining != 1 { + t.Fatalf("logical delete mismatch retained %d rows, want 1", remaining) + } +} diff --git a/internal/storage/physical_file_repository.go b/internal/storage/physical_file_repository.go index 13f3bab1..890c7c29 100644 --- a/internal/storage/physical_file_repository.go +++ b/internal/storage/physical_file_repository.go @@ -132,42 +132,47 @@ func updatePhysicalFile(ctx context.Context, ex execer, path string, logicalFile result, err := ex.ExecContext( ctx, `UPDATE physical_file - SET logical_file_id = $2, - mode = $3, - mtime = $4, - uid = $5, - gid = $6, - is_metadata_complete = $7 - WHERE path = $1`, - path, + SET logical_file_id = $1, + mode = $2, + mtime = $3, + uid = $4, + gid = $5, + is_metadata_complete = $6 + WHERE path = $7`, logicalFileID, meta.Mode, meta.MTime, meta.UID, meta.GID, meta.IsMetadataComplete, + path, ) if err != nil { return err } - _, _ = result.RowsAffected() - return nil + return db.RequireExactlyOneRow(result, "update physical file metadata") } func incrementLogicalFileRefCount(ctx context.Context, ex execer, logicalFileID int64) error { - _, err := ex.ExecContext(ctx, `UPDATE logical_file SET ref_count = ref_count + 1 WHERE id = $1`, logicalFileID) - return err + result, err := ex.ExecContext(ctx, `UPDATE logical_file SET ref_count = ref_count + 1 WHERE id = $1`, logicalFileID) + if err != nil { + return err + } + return db.RequireExactlyOneRow(result, "increment logical file refcount") } func decrementLogicalFileRefCount(ctx context.Context, ex execer, logicalFileID int64) error { - _, err := ex.ExecContext( + result, err := ex.ExecContext( ctx, `UPDATE logical_file SET ref_count = ref_count - 1 WHERE id = $1 AND ref_count > 0`, logicalFileID, ) - return err + if err != nil { + return err + } + return db.RequireExactlyOneRow(result, "decrement logical file refcount") } func ensurePhysicalFileForPathDefaultPolicyWithTx(ctx context.Context, dbconn *sql.DB, tx *sql.Tx, path string, logicalFileID int64, meta physicalFileMetadata) (bool, error) { @@ -278,7 +283,11 @@ func replacePhysicalFileLogicalTargetTx(ctx context.Context, dbconn *sql.DB, tx // Keep the delete+insert replacement strategy here because historical SQLite // behavior in this path showed unreliable persistence for in-place logical target updates. - if _, err := tx.ExecContext(ctx, `DELETE FROM physical_file WHERE path = $1`, path); err != nil { + result, err := tx.ExecContext(ctx, `DELETE FROM physical_file WHERE path = $1`, path) + if err != nil { + return fmt.Errorf("delete physical_file row for replace path %q: %w", path, err) + } + if err := db.RequireExactlyOneRow(result, "delete physical file replacement target"); err != nil { return fmt.Errorf("delete physical_file row for replace path %q: %w", path, err) } inserted, err := insertPhysicalFileIfAbsent(ctx, tx, path, newLogicalFileID, meta) diff --git a/internal/storage/remove.go b/internal/storage/remove.go index 166a4144..1ca3429f 100644 --- a/internal/storage/remove.go +++ b/internal/storage/remove.go @@ -345,11 +345,15 @@ func RemoveFileWithDBResult(dbconn *sql.DB, fileID int64) (result RemoveFileResu } // Remove logical file - _, err = tx.ExecContext(ctx, `DELETE FROM logical_file WHERE id = $1`, fileID) + deleteResult, err := tx.ExecContext(ctx, `DELETE FROM logical_file WHERE id = $1`, fileID) if err != nil { _ = tx.Rollback() return RemoveFileResult{}, err } + if err := db.RequireExactlyOneRow(deleteResult, "delete logical file"); err != nil { + _ = tx.Rollback() + return RemoveFileResult{}, err + } if err := tx.Commit(); err != nil { return RemoveFileResult{}, err diff --git a/internal/storage/storage_block_reader.go b/internal/storage/storage_block_reader.go index f8c95591..445f2df8 100644 --- a/internal/storage/storage_block_reader.go +++ b/internal/storage/storage_block_reader.go @@ -137,12 +137,13 @@ type storageVerifyContainerReader struct { func (r storageVerifyContainerReader) ReadStoredPayload(_ context.Context, meta verify.BlockStorageMetadata) ([]byte, error) { converted := &blockMetadata{ - ID: meta.BlockID, - FormatVersion: int(meta.FormatVersion), - Codec: meta.Codec, - ContainerID: meta.ContainerID, - ContainerName: meta.ContainerName, - ContainerOffset: meta.ContainerOffset, + ID: meta.BlockID, + FormatVersion: int(meta.FormatVersion), + Codec: meta.Codec, + ContainerID: meta.ContainerID, + ContainerName: meta.ContainerName, + ContainerMaxSize: meta.ContainerMaxSize, + ContainerOffset: meta.ContainerOffset, Metadata: storagemetadata.BlockStorageMetadata{ Compression: storagemetadata.CompressionMetadata{ Codec: meta.CompressionCodec, @@ -174,7 +175,7 @@ func toVerifyBlockStorageMetadata(meta *blockMetadata) verify.BlockStorageMetada ContainerID: meta.ContainerID, ContainerOffset: meta.ContainerOffset, ContainerName: meta.ContainerName, - ContainerMaxSize: container.GetContainerMaxSize(), + ContainerMaxSize: meta.ContainerMaxSize, FormatVersion: int64(meta.FormatVersion), Codec: meta.Codec, PlaintextSize: meta.Metadata.Sizes.PlaintextSize, @@ -236,14 +237,15 @@ func (r *StorageBlockReader) mapVerifyPipelineFailure(err error) error { // blockMetadata represents the persistent metadata about a block. type blockMetadata struct { - ID int64 - FormatVersion int - Codec string - Metadata storagemetadata.BlockStorageMetadata - ContainerID int64 - ContainerName string - ContainerOffset int64 - Nonce []byte + ID int64 + FormatVersion int + Codec string + Metadata storagemetadata.BlockStorageMetadata + ContainerID int64 + ContainerName string + ContainerMaxSize int64 + ContainerOffset int64 + Nonce []byte } // loadBlockMetadata queries storage_blocks and container tables to get full block metadata. @@ -257,7 +259,7 @@ func (r *StorageBlockReader) loadBlockMetadata(ctx context.Context, blockID int6 b.id, b.format_version, b.codec, b.plaintext_size, b.compression_codec, b.compression_level, b.compressed_size, b.stored_size, b.container_id, b.container_offset, b.block_hash, - b.compressed_hash, b.physical_hash, c.filename + b.compressed_hash, b.physical_hash, c.filename, c.max_size FROM storage_blocks b JOIN container c ON b.container_id = c.id WHERE b.id = $1 @@ -287,6 +289,7 @@ func (r *StorageBlockReader) loadBlockMetadata(ctx context.Context, blockID int6 &compressedHash, &physicalHash, &meta.ContainerName, + &meta.ContainerMaxSize, ) if err == sql.ErrNoRows { return nil, fmt.Errorf("block %d not found", blockID) @@ -355,9 +358,9 @@ func (r *StorageBlockReader) readStoredPayload(meta *blockMetadata) ([]byte, err return nil, fmt.Errorf("invalid container filename %q: %w", meta.ContainerName, err) } - // Open container file for reading - // Note: We use the maximum container size here; this is just for validation purposes - fc, err := container.OpenReadOnlyContainer(containerPath, container.GetContainerMaxSize()) + // Open against the maximum persisted with this container. The process-global + // setting may have changed since the repository artifact was created. + fc, err := container.OpenReadOnlyContainer(containerPath, meta.ContainerMaxSize) if err != nil { return nil, fmt.Errorf("open container %s: %w", meta.ContainerName, err) } diff --git a/internal/storage/storage_block_reader_test.go b/internal/storage/storage_block_reader_test.go index 17e9ccb5..5bb9ee18 100644 --- a/internal/storage/storage_block_reader_test.go +++ b/internal/storage/storage_block_reader_test.go @@ -296,6 +296,9 @@ func TestStorageBlockReaderLoadBlockMetadataIncludesTransformAwareFields(t *test if meta.Metadata.Sizes.PlaintextSize != 100 { t.Fatalf("plaintext size: got %d want %d", meta.Metadata.Sizes.PlaintextSize, 100) } + if meta.ContainerMaxSize != 1048576 { + t.Fatalf("container max size: got %d want %d", meta.ContainerMaxSize, 1048576) + } } func TestStorageBlockReaderLoadBlockMetadataRejectsInvalidCompressionContract(t *testing.T) { @@ -310,7 +313,8 @@ func TestStorageBlockReaderLoadBlockMetadataRejectsInvalidCompressionContract(t if _, err := dbconn.ExecContext(context.Background(), ` CREATE TABLE container ( id INTEGER PRIMARY KEY, - filename TEXT NOT NULL + filename TEXT NOT NULL, + max_size INTEGER NOT NULL ); `); err != nil { t.Fatalf("create container table: %v", err) @@ -335,7 +339,7 @@ func TestStorageBlockReaderLoadBlockMetadataRejectsInvalidCompressionContract(t t.Fatalf("create storage_blocks table: %v", err) } - if _, err := dbconn.ExecContext(context.Background(), `INSERT INTO container (id, filename) VALUES (1, 'test.bin')`); err != nil { + if _, err := dbconn.ExecContext(context.Background(), `INSERT INTO container (id, filename, max_size) VALUES (1, 'test.bin', 1048576)`); err != nil { t.Fatalf("insert container row: %v", err) } if _, err := dbconn.ExecContext(context.Background(), ` @@ -365,6 +369,51 @@ func setupStoredBlockForReaderCorruption(t *testing.T, codec blocks.Codec) (*sql return dbconn, workDir, blockID, containerFilename, offset, storedSize } +func TestStorageBlockReaderUsesCatalogContainerMaxSize(t *testing.T) { + dbconn, workDir, blockID, _, _, _ := setupStoredBlockForReaderCorruption(t, blocks.CodecPlain) + + var catalogMaxSize int64 + if err := dbconn.QueryRow(` + SELECT c.max_size + FROM storage_blocks sb + JOIN container c ON c.id = sb.container_id + WHERE sb.id = $1 + `, blockID).Scan(&catalogMaxSize); err != nil { + t.Fatalf("load catalog container maximum: %v", err) + } + + originalMaxSize := container.GetContainerMaxSize() + changedMaxSize := catalogMaxSize / 2 + if changedMaxSize <= container.ContainerHdrLen { + changedMaxSize = catalogMaxSize + 1 + } + container.SetContainerMaxSize(changedMaxSize) + t.Cleanup(func() { container.SetContainerMaxSize(originalMaxSize) }) + + reader := NewStorageBlockReader(dbconn, workDir) + if _, err := reader.ReadBlock(context.Background(), blockID); err != nil { + t.Fatalf("read block with process-global maximum %d and catalog maximum %d: %v", changedMaxSize, catalogMaxSize, err) + } +} + +func TestStorageBlockReaderRejectsContainerHeaderCatalogMaxSizeMismatch(t *testing.T) { + dbconn, workDir, blockID, _, _, _ := setupStoredBlockForReaderCorruption(t, blocks.CodecPlain) + + if _, err := dbconn.Exec(` + UPDATE container + SET max_size = max_size + 1 + WHERE id = (SELECT container_id FROM storage_blocks WHERE id = $1) + `, blockID); err != nil { + t.Fatalf("mutate catalog container maximum: %v", err) + } + + reader := NewStorageBlockReader(dbconn, workDir) + _, err := reader.ReadBlock(context.Background(), blockID) + if err == nil || !strings.Contains(err.Error(), "container max size mismatch") { + t.Fatalf("expected header/catalog maximum mismatch, got: %v", err) + } +} + func setupStoredBlockFixtureForReaderCorruption(t *testing.T, codec blocks.Codec, compressionCodec string) (int64, *sql.DB, string, int64, string, int64, int64) { t.Helper() compressionCodec = strings.TrimSpace(strings.ToLower(compressionCodec)) @@ -535,13 +584,17 @@ func TestStorageBlockReaderPhysicalHashMismatchOnTruncatedPayload(t *testing.T) } func TestStorageBlockReaderPhysicalHashMismatchOnWrongOffset(t *testing.T) { - dbconn, workDir, blockID, _, offset, _ := setupStoredBlockForReaderCorruption(t, blocks.CodecPlain) + dbconn, workDir, blockID, _, offset, storedSize := setupStoredBlockForReaderCorruption(t, blocks.CodecPlain) - if offset <= 0 { - t.Fatalf("unexpected non-positive offset for wrong-offset test: %d", offset) + if storedSize <= 1 { + t.Fatalf("unexpected stored size for wrong-offset test: %d", storedSize) } - if _, err := dbconn.Exec(`UPDATE storage_blocks SET container_offset = $1 WHERE id = $2`, offset-1, blockID); err != nil { + if _, err := dbconn.Exec(` + UPDATE storage_blocks + SET container_offset = $1, stored_size = $2 + WHERE id = $3 + `, offset+1, storedSize-1, blockID); err != nil { t.Fatalf("update container_offset: %v", err) } @@ -900,7 +953,7 @@ func TestStorageBlockReaderDecompressionFailureAfterCompressedHashFixtureUpdate( assertReaderCorruptionRestoreFailsWithoutOutput(t, dbconn, fileID, workDir, "compressed-decompression-failure.restore", "decompress codec=zstd") } -func TestStorageBlockReaderEncryptedPlaintextSizeMismatchDetectedWithoutPartialOutput(t *testing.T) { +func TestStorageBlockReaderRejectsZstdOutputBeyondExpectedSizeAfterAESGCMDecrypt(t *testing.T) { t.Setenv("COLDKEEP_KEY", strings.Repeat("ab", 32)) fileID, dbconn, workDir, blockID, _, _, _ := setupStoredBlockFixtureForReaderCorruption(t, blocks.CodecAESGCM, storagecompression.CompressionZstd) @@ -930,6 +983,45 @@ func TestStorageBlockReaderEncryptedPlaintextSizeMismatchDetectedWithoutPartialO } } +func TestRestorePackedZstdBoundFailurePreservesDestinationAndCleansTemp(t *testing.T) { + t.Setenv("COLDKEEP_KEY", strings.Repeat("ab", 32)) + fileID, dbconn, workDir, blockID, _, _, _ := setupStoredBlockFixtureForReaderCorruption(t, blocks.CodecAESGCM, storagecompression.CompressionZstd) + + if _, err := dbconn.Exec(`UPDATE storage_blocks SET plaintext_size = $1 WHERE id = $2`, int64(1), blockID); err != nil { + t.Fatalf("update plaintext_size for bound fixture: %v", err) + } + + outputDir := t.TempDir() + outPath := filepath.Join(outputDir, "bounded-restore.bin") + original := []byte("existing-destination-must-survive") + if err := os.WriteFile(outPath, original, 0o600); err != nil { + t.Fatalf("write existing destination: %v", err) + } + + _, err := restoreFileWithDBAndDir(dbconn, fileID, outPath, workDir, RestoreOptions{Overwrite: true}) + if err == nil || (!strings.Contains(err.Error(), "decompress codec=\"zstd\"") && !strings.Contains(err.Error(), "decompress codec=zstd")) { + t.Fatalf("expected bounded zstd restore failure, got: %v", err) + } + + got, readErr := os.ReadFile(outPath) + if readErr != nil { + t.Fatalf("read preserved destination: %v", readErr) + } + if !bytes.Equal(got, original) { + t.Fatalf("destination changed after bound failure: got=%q want=%q", got, original) + } + + entries, readDirErr := os.ReadDir(outputDir) + if readDirErr != nil { + t.Fatalf("read output directory: %v", readDirErr) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".coldkeep-restore-") { + t.Fatalf("stale restore temporary file: %s", entry.Name()) + } + } +} + func TestStorageBlockReaderEncryptedLogicalHashMismatchBeforeDecode(t *testing.T) { t.Setenv("COLDKEEP_KEY", strings.Repeat("ab", 32)) fileID, dbconn, workDir, blockID, _, _, _ := setupStoredBlockFixtureForReaderCorruption(t, blocks.CodecAESGCM, storagecompression.CompressionZstd) diff --git a/internal/storage/store.go b/internal/storage/store.go index b02ebca7..302e1f18 100644 --- a/internal/storage/store.go +++ b/internal/storage/store.go @@ -36,6 +36,8 @@ type payloadStatefulWriter interface { type storeInterleavingEvent string +var errSharedPackedBlockPartialRebuild = errors.New("partial rebuild of shared packed block refused") + const ( storeInterleavingEventAfterChunkClaim storeInterleavingEvent = "after_chunk_claim" storeInterleavingEventBeforePackedFlush storeInterleavingEvent = "before_packed_flush" @@ -422,12 +424,20 @@ func commitPreparedChunksWithContext( return err } - if _, err := tx.ExecContext( + result, err := tx.ExecContext( ctx, `UPDATE chunk SET status = $1 WHERE id = $2`, filestate.ChunkCompleted, pending.chunkID, - ); err != nil { + ) + if err != nil { + _ = tx.Rollback() + if rbErr := rollbackWriterLastAppendWithQuarantine(writer); rbErr != nil { + return errors.Join(err, rbErr) + } + return err + } + if err := db.RequireExactlyOneRow(result, "complete packed chunk"); err != nil { _ = tx.Rollback() if rbErr := rollbackWriterLastAppendWithQuarantine(writer); rbErr != nil { return errors.Join(err, rbErr) @@ -680,7 +690,12 @@ func commitPreparedChunksWithContext( return StoreFileResult{}, err } - if _, err := tx.ExecContext(ctx, `UPDATE chunk SET status = $1 WHERE id = $2`, filestate.ChunkCompleted, claimedChunkID); err != nil { + result, err := tx.ExecContext(ctx, `UPDATE chunk SET status = $1 WHERE id = $2`, filestate.ChunkCompleted, claimedChunkID) + if err != nil { + _ = tx.Rollback() + return StoreFileResult{}, err + } + if err := db.RequireExactlyOneRow(result, "complete reclaimed chunk"); err != nil { _ = tx.Rollback() return StoreFileResult{}, err } @@ -909,7 +924,11 @@ func markContainerSealingInTx(tx *sql.Tx, containerID int64) error { if tx == nil || containerID <= 0 { return nil } - if _, err := tx.Exec(`UPDATE container SET sealing = TRUE WHERE id = $1`, containerID); err != nil { + result, err := tx.Exec(`UPDATE container SET sealing = TRUE WHERE id = $1`, containerID) + if err != nil { + return fmt.Errorf("mark container %d sealing in tx: %w", containerID, err) + } + if err := db.RequireExactlyOneRow(result, "mark container sealing"); err != nil { return fmt.Errorf("mark container %d sealing in tx: %w", containerID, err) } return nil @@ -1514,6 +1533,96 @@ func validateReusableChunkCompanionMappingWithContext(ctx context.Context, dbcon } } +func lockAndValidateChunkRebuildCandidatesWithContext( + ctx context.Context, + dbconn *sql.DB, + tx *sql.Tx, + chunkID int64, +) ([]int64, error) { + candidateQuery := db.QueryWithOptionalForUpdate(dbconn, ` + SELECT sb.id + FROM storage_blocks sb + JOIN chunk_block_refs target ON target.block_id = sb.id + WHERE target.chunk_id = $1 + ORDER BY sb.id + `) + rows, err := tx.QueryContext(ctx, candidateQuery, chunkID) + if err != nil { + return nil, fmt.Errorf("query and lock storage_blocks for chunk %d rebuild cleanup: %w", chunkID, err) + } + candidateBlockIDs, err := scanLockedBlockIDs(rows, chunkID) + if err != nil { + return nil, err + } + + memberQuery := db.QueryWithOptionalForUpdate(dbconn, ` + SELECT chunk_id + FROM chunk_block_refs + WHERE block_id = $1 + ORDER BY chunk_id + `) + for _, blockID := range candidateBlockIDs { + memberRows, err := tx.QueryContext(ctx, memberQuery, blockID) + if err != nil { + return nil, fmt.Errorf("query and lock members for storage_block %d during chunk %d rebuild cleanup: %w", blockID, chunkID, err) + } + memberCount, err := countLockedBlockMembers(memberRows, blockID, chunkID) + if err != nil { + return nil, err + } + if memberCount > 1 { + return nil, fmt.Errorf( + "cannot rebuild chunk %d independently: packed block %d has %d active members: %w", + chunkID, + blockID, + memberCount, + errSharedPackedBlockPartialRebuild, + ) + } + } + return candidateBlockIDs, nil +} + +func scanLockedBlockIDs(rows *sql.Rows, chunkID int64) ([]int64, error) { + var blockIDs []int64 + for rows.Next() { + var blockID int64 + if err := rows.Scan(&blockID); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan storage_block id for chunk %d rebuild cleanup: %w", chunkID, err) + } + blockIDs = append(blockIDs, blockID) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("iterate storage_block ids for chunk %d rebuild cleanup: %w", chunkID, err) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close storage_block ids rows for chunk %d rebuild cleanup: %w", chunkID, err) + } + return blockIDs, nil +} + +func countLockedBlockMembers(rows *sql.Rows, blockID, chunkID int64) (int, error) { + count := 0 + for rows.Next() { + var memberChunkID int64 + if err := rows.Scan(&memberChunkID); err != nil { + _ = rows.Close() + return 0, fmt.Errorf("scan member for storage_block %d during chunk %d rebuild cleanup: %w", blockID, chunkID, err) + } + count++ + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, fmt.Errorf("iterate members for storage_block %d during chunk %d rebuild cleanup: %w", blockID, chunkID, err) + } + if err := rows.Close(); err != nil { + return 0, fmt.Errorf("close members for storage_block %d during chunk %d rebuild cleanup: %w", blockID, chunkID, err) + } + return count, nil +} + func markChunkForRebuildWithContext(ctx context.Context, dbconn *sql.DB, chunkID int64) error { state := storeInterleavingStateFromContext(ctx) storeOpID := "" @@ -1539,25 +1648,9 @@ func markChunkForRebuildWithContext(ctx context.Context, dbconn *sql.DB, chunkID } defer func() { _ = tx.Rollback() }() - rows, err := tx.QueryContext(ctx, `SELECT block_id FROM chunk_block_refs WHERE chunk_id = $1`, chunkID) + candidateBlockIDs, err := lockAndValidateChunkRebuildCandidatesWithContext(ctx, dbconn, tx, chunkID) if err != nil { - return fmt.Errorf("query storage_blocks for chunk %d rebuild cleanup: %w", chunkID, err) - } - var candidateBlockIDs []int64 - for rows.Next() { - var blockID int64 - if err := rows.Scan(&blockID); err != nil { - _ = rows.Close() - return fmt.Errorf("scan storage_block id for chunk %d rebuild cleanup: %w", chunkID, err) - } - candidateBlockIDs = append(candidateBlockIDs, blockID) - } - if err := rows.Err(); err != nil { - _ = rows.Close() - return fmt.Errorf("iterate storage_block ids for chunk %d rebuild cleanup: %w", chunkID, err) - } - if err := rows.Close(); err != nil { - return fmt.Errorf("close storage_block ids rows for chunk %d rebuild cleanup: %w", chunkID, err) + return err } result, err := tx.ExecContext(ctx, @@ -2602,7 +2695,11 @@ func linkFileChunkWithContext(ctx context.Context, tx *sql.Tx, fileID int64, chu } if rowsAffected > 0 && incrementRefCount { - if _, err := tx.ExecContext(ctx, `UPDATE chunk SET live_ref_count = live_ref_count + 1 WHERE id = $1`, chunkID); err != nil { + result, err := tx.ExecContext(ctx, `UPDATE chunk SET live_ref_count = live_ref_count + 1 WHERE id = $1`, chunkID) + if err != nil { + return err + } + if err := db.RequireExactlyOneRow(result, "increment linked chunk live refcount"); err != nil { return err } } @@ -2651,12 +2748,16 @@ func finalizeLogicalFileStorageWithContext(ctx context.Context, dbconn *sql.DB, } // All verification passed; mark file complete in the same transaction. - if _, err := tx.ExecContext( + result, err := tx.ExecContext( ctx, `UPDATE logical_file SET status = $1 WHERE id = $2`, filestate.LogicalFileCompleted, fileID, - ); err != nil { + ) + if err != nil { + return fmt.Errorf("update logical_file to COMPLETED: %w", err) + } + if err := db.RequireExactlyOneRow(result, "finalize logical file storage"); err != nil { return fmt.Errorf("update logical_file to COMPLETED: %w", err) } diff --git a/internal/storage/store_interleaving_rebuild_test.go b/internal/storage/store_interleaving_rebuild_test.go index b4f7978c..dbd4ee22 100644 --- a/internal/storage/store_interleaving_rebuild_test.go +++ b/internal/storage/store_interleaving_rebuild_test.go @@ -4,13 +4,17 @@ import ( "bytes" "context" "database/sql" + "errors" + "fmt" "path/filepath" + "slices" "strings" "testing" "github.com/franchoy/coldkeep/internal/chunk" "github.com/franchoy/coldkeep/internal/db" filestate "github.com/franchoy/coldkeep/internal/status" + verifypkg "github.com/franchoy/coldkeep/internal/verify" ) func TestStoreInterleavingRebuildCleanupDeletesBothMappings(t *testing.T) { @@ -117,13 +121,410 @@ func assertRebuildCleanupHookSequence(t *testing.T, events []TestStoreInterleavi func TestStoreInterleavingRebuildCleanupRetainsSharedStorageBlock(t *testing.T) { dbconn, _ := openSharedStoreInterleavingDB(t) containersDir := t.TempDir() - losingChunkID, blockID, winningHash := seedSharedRebuildCleanupFixture(t, dbconn, containersDir) - if err := markChunkForRebuildWithContext(context.Background(), dbconn, losingChunkID); err != nil { - t.Fatalf("mark losing chunk for rebuild: %v", err) + losingChunkID, blockID, _ := seedSharedRebuildCleanupFixture(t, dbconn, containersDir) + err := markChunkForRebuildWithContext(context.Background(), dbconn, losingChunkID) + if !errors.Is(err, errSharedPackedBlockPartialRebuild) { + t.Fatalf("expected shared packed-block rebuild refusal, got %v", err) } - assertChunkMappingsRemoved(t, dbconn, losingChunkID) assertSharedStorageBlockRetained(t, dbconn, blockID) - assertSharedRebuildSurvivorState(t, dbconn, winningHash, losingChunkID) + packedRows := scanInterleavingCount(t, "count selected packed rows after refusal", dbconn.QueryRow( + `SELECT COUNT(*) FROM chunk_block_refs WHERE chunk_id = $1 AND block_id = $2`, losingChunkID, blockID, + )) + if packedRows != 1 { + t.Fatalf("expected selected shared-block mapping to remain after refusal, got %d rows", packedRows) + } + var losingStatus string + if err := dbconn.QueryRow(`SELECT status FROM chunk WHERE id = $1`, losingChunkID).Scan(&losingStatus); err != nil { + t.Fatalf("load selected chunk status after refusal: %v", err) + } + if losingStatus != filestate.ChunkCompleted { + t.Fatalf("expected selected chunk status to remain COMPLETED, got %s", losingStatus) + } +} + +func TestSharedPackedBlockSingleMemberRebuildCannotLeavePartialMembership(t *testing.T) { + dbconn, _ := openSharedStoreInterleavingDB(t) + containersDir := t.TempDir() + selectedPayload := bytes.Repeat([]byte("S"), 64) + siblingPayload := bytes.Repeat([]byte("W"), 64) + seeds := seedPackedFixtureBlock( + t, + dbconn, + containersDir, + packedFixtureChunkSpec{ + hash: storeInterleavingHash(selectedPayload), + payload: selectedPayload, + liveRefCount: 0, + withCompanion: true, + }, + packedFixtureChunkSpec{ + hash: storeInterleavingHash(siblingPayload), + payload: siblingPayload, + liveRefCount: 0, + withCompanion: true, + }, + ) + if len(seeds) != 2 || seeds[0].blockID != seeds[1].blockID { + t.Fatalf("fixture must create exactly two members in one packed block: %+v", seeds) + } + + blockID := seeds[0].blockID + memberIDs := []int64{seeds[0].chunkID, seeds[1].chunkID} + slices.Sort(memberIDs) + before := loadSharedPackedBlockSnapshot(t, dbconn, containersDir, blockID, memberIDs) + assertValidSharedPackedBlockSnapshot(t, before, memberIDs) + selectedCompanionBefore := requireValidSharedPackedCompanion(t, dbconn, seeds[0].chunkID, "selected before rebuild") + siblingCompanionBefore := requireValidSharedPackedCompanion(t, dbconn, seeds[1].chunkID, "sibling before rebuild") + if err := verifypkg.VerifyRepository(dbconn, containersDir); err != nil { + t.Fatalf("baseline shared packed block must pass full verification: %v", err) + } + + var events []TestStoreInterleavingHookEvent + ctx := withStoreInterleavingState(context.Background(), &storeInterleavingState{ + hooks: &storeInterleavingHooks{ + onEvent: func(_ context.Context, event storeInterleavingHookEvent) error { + events = append(events, event) + return nil + }, + }, + storeOpID: "shared-packed-single-member-rebuild", + codec: "plain", + fileHash: seeds[0].hash, + }) + rebuildErr := markChunkForRebuildWithContext(ctx, dbconn, seeds[0].chunkID) + if rebuildErr == nil { + t.Fatal("shared packed-block single-member rebuild must fail closed") + } + if !errors.Is(rebuildErr, errSharedPackedBlockPartialRebuild) { + t.Fatalf("expected classified shared packed-block rebuild refusal, got %v", rebuildErr) + } + wantMessage := fmt.Sprintf( + "cannot rebuild chunk %d independently: packed block %d has 2 active members", + seeds[0].chunkID, + blockID, + ) + if !strings.Contains(rebuildErr.Error(), wantMessage) { + t.Fatalf("expected stable shared packed-block refusal %q, got %v", wantMessage, rebuildErr) + } + assertSharedRebuildRefusalHookSequence(t, events, seeds[0].chunkID) + + after := loadSharedPackedBlockSnapshot(t, dbconn, containersDir, blockID, memberIDs) + selectedCompanionAfter := requireValidSharedPackedCompanion(t, dbconn, seeds[0].chunkID, "selected after refusal") + siblingCompanionAfter := requireValidSharedPackedCompanion(t, dbconn, seeds[1].chunkID, "sibling after refusal") + if !sharedPackedBlockSnapshotsEqual(before, after) { + t.Fatalf("shared packed-block refusal mutated repository state: before=%+v after=%+v", before, after) + } + if !selectedCompanionBefore || !siblingCompanionBefore || !selectedCompanionAfter || !siblingCompanionAfter { + t.Fatalf( + "shared packed-block companions must remain valid after refusal: selected_before=%t sibling_before=%t selected_after=%t sibling_after=%t", + selectedCompanionBefore, + siblingCompanionBefore, + selectedCompanionAfter, + siblingCompanionAfter, + ) + } + if err := verifypkg.VerifyRepository(dbconn, containersDir); err != nil { + t.Fatalf("shared packed block must remain fully verifiable after refusal: %v", err) + } +} + +func TestStoreInterleavingRebuildCleanupAllowsSingleMemberPackedBlock(t *testing.T) { + dbconn, _ := openSharedStoreInterleavingDB(t) + containersDir := t.TempDir() + payload := bytes.Repeat([]byte("S"), 64) + chunkHash := storeInterleavingHash(payload) + seeds := seedPackedFixtureBlock(t, dbconn, containersDir, packedFixtureChunkSpec{ + hash: chunkHash, + payload: payload, + liveRefCount: 0, + withCompanion: true, + }) + if len(seeds) != 1 { + t.Fatalf("fixture must create exactly one packed member: %+v", seeds) + } + if err := verifypkg.VerifyRepository(dbconn, containersDir); err != nil { + t.Fatalf("baseline single-member packed block must pass full verification: %v", err) + } + + if err := markChunkForRebuildWithContext(context.Background(), dbconn, seeds[0].chunkID); err != nil { + t.Fatalf("single-member packed block rebuild cleanup: %v", err) + } + assertInterleavingChunkFinalState(t, dbconn, containersDir, chunkHash, len(payload), interleavingChunkFinalState{ + chunkStatus: filestate.ChunkAborted, + }) + blockRows := scanInterleavingCount(t, "count single-member storage block after cleanup", dbconn.QueryRow( + `SELECT COUNT(*) FROM storage_blocks WHERE id = $1`, seeds[0].blockID, + )) + if blockRows != 0 { + t.Fatalf("expected orphaned single-member storage block to be deleted, got %d rows", blockRows) + } + if err := verifypkg.VerifyRepository(dbconn, containersDir); err != nil { + t.Fatalf("single-member rebuild cleanup must leave repository verifiable: %v", err) + } +} + +type sharedPackedBlockSnapshot struct { + blockExists bool + relationalMembers []int64 + companionMembers []int64 + encodedMembers []int64 + chunkStatuses []string + storageMetadata string + physicalFileRows int + physicalVerified bool + physicalError string +} + +func loadSharedPackedBlockSnapshot( + t *testing.T, + dbconn *sql.DB, + containersDir string, + blockID int64, + memberIDs []int64, +) sharedPackedBlockSnapshot { + t.Helper() + if len(memberIDs) != 2 { + t.Fatalf("shared packed block diagnostic expects two members, got %v", memberIDs) + } + + snapshot := sharedPackedBlockSnapshot{ + relationalMembers: loadSharedPackedBlockMemberIDs(t, dbconn, blockID), + companionMembers: loadSharedPackedCompanionMemberIDs(t, dbconn, memberIDs), + chunkStatuses: loadSharedPackedChunkStatuses(t, dbconn, memberIDs), + physicalFileRows: scanInterleavingCount(t, "count physical-file metadata for shared block", dbconn.QueryRow( + `SELECT COUNT(*) FROM physical_file`, + )), + } + + var meta verifypkg.BlockStorageMetadata + var compressionLevel sql.NullInt64 + var compressedSize sql.NullInt64 + var containerCurrentSize int64 + err := dbconn.QueryRow(` + SELECT sb.format_version, sb.codec, sb.compression_codec, sb.compression_level, + sb.plaintext_size, sb.compressed_size, sb.stored_size, + sb.container_id, c.filename, c.current_size, c.max_size, sb.container_offset, + sb.block_hash, sb.compressed_hash, sb.physical_hash + FROM storage_blocks sb + JOIN container c ON c.id = sb.container_id + WHERE sb.id = $1 + `, blockID).Scan( + &meta.FormatVersion, + &meta.Codec, + &meta.CompressionCodec, + &compressionLevel, + &meta.PlaintextSize, + &compressedSize, + &meta.StoredSize, + &meta.ContainerID, + &meta.ContainerName, + &containerCurrentSize, + &meta.ContainerMaxSize, + &meta.ContainerOffset, + &meta.LogicalHash, + &meta.CompressedHash, + &meta.PhysicalHash, + ) + if err == sql.ErrNoRows { + return snapshot + } + if err != nil { + t.Fatalf("load shared packed block %d metadata: %v", blockID, err) + } + snapshot.blockExists = true + meta.BlockID = blockID + snapshot.storageMetadata = fmt.Sprintf( + "container_id=%d name=%s current_size=%d max_size=%d offset=%d stored_size=%d logical_hash=%x compressed_hash=%x physical_hash=%x", + meta.ContainerID, + meta.ContainerName, + containerCurrentSize, + meta.ContainerMaxSize, + meta.ContainerOffset, + meta.StoredSize, + meta.LogicalHash, + meta.CompressedHash, + meta.PhysicalHash, + ) + if compressedSize.Valid { + value := compressedSize.Int64 + meta.CompressedSize = &value + } + if compressionLevel.Valid { + value := int(compressionLevel.Int64) + meta.CompressionLevel = &value + } + + verified, err := verifypkg.VerifyStoredBlock( + context.Background(), + meta, + verifypkg.FilesystemContainerReader{ContainersDir: containersDir}, + ) + if err != nil { + snapshot.physicalError = err.Error() + return snapshot + } + snapshot.physicalVerified = true + for _, entry := range verified.DecodedBlock.Entries { + snapshot.encodedMembers = append(snapshot.encodedMembers, int64(entry.ChunkID)) + } + slices.Sort(snapshot.encodedMembers) + return snapshot +} + +func loadSharedPackedChunkStatuses(t *testing.T, dbconn *sql.DB, memberIDs []int64) []string { + t.Helper() + rows, err := dbconn.Query( + `SELECT status FROM chunk WHERE id = $1 OR id = $2 ORDER BY id`, + memberIDs[0], + memberIDs[1], + ) + if err != nil { + t.Fatalf("load shared packed chunk statuses: %v", err) + } + defer func() { _ = rows.Close() }() + var statuses []string + for rows.Next() { + var status string + if err := rows.Scan(&status); err != nil { + t.Fatalf("scan shared packed chunk status: %v", err) + } + statuses = append(statuses, status) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate shared packed chunk statuses: %v", err) + } + return statuses +} + +func loadSharedPackedBlockMemberIDs(t *testing.T, dbconn *sql.DB, blockID int64) []int64 { + t.Helper() + rows, err := dbconn.Query(`SELECT chunk_id FROM chunk_block_refs WHERE block_id = $1 ORDER BY chunk_id`, blockID) + if err != nil { + t.Fatalf("load relational members for packed block %d: %v", blockID, err) + } + defer func() { _ = rows.Close() }() + return scanSharedPackedMemberIDs(t, rows, "relational packed members") +} + +func loadSharedPackedCompanionMemberIDs(t *testing.T, dbconn *sql.DB, memberIDs []int64) []int64 { + t.Helper() + rows, err := dbconn.Query( + `SELECT chunk_id FROM blocks WHERE chunk_id = $1 OR chunk_id = $2 ORDER BY chunk_id`, + memberIDs[0], + memberIDs[1], + ) + if err != nil { + t.Fatalf("load packed companion members: %v", err) + } + defer func() { _ = rows.Close() }() + return scanSharedPackedMemberIDs(t, rows, "packed companion members") +} + +func scanSharedPackedMemberIDs(t *testing.T, rows *sql.Rows, label string) []int64 { + t.Helper() + var memberIDs []int64 + for rows.Next() { + var chunkID int64 + if err := rows.Scan(&chunkID); err != nil { + t.Fatalf("scan %s: %v", label, err) + } + memberIDs = append(memberIDs, chunkID) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate %s: %v", label, err) + } + return memberIDs +} + +func assertValidSharedPackedBlockSnapshot(t *testing.T, snapshot sharedPackedBlockSnapshot, memberIDs []int64) { + t.Helper() + if !snapshot.blockExists || !snapshot.physicalVerified || snapshot.physicalError != "" { + t.Fatalf("fixture packed block physical metadata/bytes are invalid: %+v", snapshot) + } + if len(memberIDs) < 2 || !slices.Equal(snapshot.relationalMembers, memberIDs) || !slices.Equal(snapshot.companionMembers, memberIDs) || !slices.Equal(snapshot.encodedMembers, memberIDs) { + t.Fatalf("fixture must begin with identical multi-member relational, companion, and encoded membership: want=%v got=%+v", memberIDs, snapshot) + } +} + +func requireValidSharedPackedCompanion(t *testing.T, dbconn *sql.DB, chunkID int64, label string) bool { + t.Helper() + valid, err := validateReusableChunkCompanionMappingWithContext(context.Background(), dbconn, chunkID) + if err != nil { + t.Fatalf("%s companion validation: %v", label, err) + } + if !valid { + t.Fatalf("%s companion must be valid: %s", label, describeSharedPackedCompanion(t, dbconn, chunkID)) + } + return valid +} + +func describeSharedPackedCompanion(t *testing.T, dbconn *sql.DB, chunkID int64) string { + t.Helper() + var chunkSize, plaintextSize, storedSize, legacyContainerID, legacyOffset int64 + var offsetInBlock, sizeInBlock, packedContainerID, packedContainerOffset, packedPlaintextSize, totalReferencedBytes int64 + var codec string + if err := dbconn.QueryRow(` + SELECT c.size, b.codec, b.plaintext_size, b.stored_size, b.container_id, b.block_offset, + r.offset_in_block, r.size_in_block, sb.container_id, sb.container_offset, + sb.plaintext_size, + (SELECT COALESCE(SUM(size_in_block), 0) FROM chunk_block_refs WHERE block_id = r.block_id) + FROM chunk c + JOIN blocks b ON b.chunk_id = c.id + JOIN chunk_block_refs r ON r.chunk_id = c.id + JOIN storage_blocks sb ON sb.id = r.block_id + WHERE c.id = $1 + `, chunkID).Scan( + &chunkSize, + &codec, + &plaintextSize, + &storedSize, + &legacyContainerID, + &legacyOffset, + &offsetInBlock, + &sizeInBlock, + &packedContainerID, + &packedContainerOffset, + &packedPlaintextSize, + &totalReferencedBytes, + ); err != nil { + return err.Error() + } + return fmt.Sprintf( + "chunk_size=%d codec=%s plaintext_size=%d stored_size=%d legacy_container=%d legacy_offset=%d offset_in_block=%d size_in_block=%d packed_container=%d packed_offset=%d packed_plaintext_size=%d total_referenced=%d derived_prefix=%d expected_legacy_offset=%d", + chunkSize, + codec, + plaintextSize, + storedSize, + legacyContainerID, + legacyOffset, + offsetInBlock, + sizeInBlock, + packedContainerID, + packedContainerOffset, + packedPlaintextSize, + totalReferencedBytes, + packedPlaintextSize-totalReferencedBytes, + packedContainerOffset+packedPlaintextSize-totalReferencedBytes+offsetInBlock, + ) +} + +func sharedPackedBlockSnapshotsEqual(left, right sharedPackedBlockSnapshot) bool { + return left.blockExists == right.blockExists && + slices.Equal(left.relationalMembers, right.relationalMembers) && + slices.Equal(left.companionMembers, right.companionMembers) && + slices.Equal(left.encodedMembers, right.encodedMembers) && + slices.Equal(left.chunkStatuses, right.chunkStatuses) && + left.storageMetadata == right.storageMetadata && + left.physicalFileRows == right.physicalFileRows && + left.physicalVerified == right.physicalVerified && + left.physicalError == right.physicalError +} + +func assertSharedRebuildRefusalHookSequence(t *testing.T, events []TestStoreInterleavingHookEvent, chunkID int64) { + t.Helper() + if len(events) != 1 || events[0].Event != TestStoreInterleavingEventBeforeMarkChunkForRebuild || events[0].ChunkID != chunkID { + t.Fatalf("expected rebuild refusal before-mutation hook only, got %+v", events) + } } func seedSharedRebuildCleanupFixture(t *testing.T, dbconn *sql.DB, containersDir string) (int64, int64, string) { @@ -162,40 +563,6 @@ func assertSharedStorageBlockRetained(t *testing.T, dbconn *sql.DB, blockID int6 } } -func assertSharedRebuildSurvivorState(t *testing.T, dbconn *sql.DB, winningHash string, losingChunkID int64) { - t.Helper() - // This synthetic fixture intentionally seeds one packed payload that still - // encodes both chunks, then removes only one chunk_block_refs row. That - // leaves the surviving storage_blocks row still referenced, which is the - // cleanup-retention property under test, but the packed payload can no longer - // satisfy full payload-level verification for the deleted chunk entry. - gotWinner := loadInterleavingChunkFinalState(t, dbconn, winningHash, 64) - wantWinner := interleavingChunkFinalState{ - chunkStatus: filestate.ChunkCompleted, - packedMappings: 1, - legacyMappings: 0, - logicalFileRefs: 0, - physicalFileRefs: 0, - storageBlockRefs: 1, - storageBlocks: 1, - orphanStorageBlocks: 0, - sealedContainers: 0, - quarantinedConts: 0, - validCompanionState: false, - } - if comparableInterleavingState(gotWinner) != comparableInterleavingState(wantWinner) { - t.Fatalf("unexpected shared-reference survivor state: got=%+v want=%+v", gotWinner, wantWinner) - } - - var losingStatus string - if err := dbconn.QueryRow(`SELECT status FROM chunk WHERE id = $1`, losingChunkID).Scan(&losingStatus); err != nil { - t.Fatalf("load losing chunk status: %v", err) - } - if losingStatus != filestate.ChunkAborted { - t.Fatalf("expected losing chunk status ABORTED, got %s", losingStatus) - } -} - func TestStoreInterleavingRebuildCleanupRollbackRestoresRows(t *testing.T) { dbconn, _ := openSharedStoreInterleavingDB(t) containersDir := t.TempDir() diff --git a/internal/storage/store_test.go b/internal/storage/store_test.go index dc55bd41..2192437a 100644 --- a/internal/storage/store_test.go +++ b/internal/storage/store_test.go @@ -23,6 +23,7 @@ import ( corebenchmark "github.com/franchoy/coldkeep/internal/benchmark" "github.com/franchoy/coldkeep/internal/blocks" "github.com/franchoy/coldkeep/internal/chunk" + "github.com/franchoy/coldkeep/internal/chunk/fastcdc" "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/execution" gcpkg "github.com/franchoy/coldkeep/internal/gc" @@ -2651,16 +2652,6 @@ func setupCrossVersionSharedChunkScenario(t *testing.T) crossVersionSharedChunkS t.Fatalf("run migrations: %v", err) } - if _, err := dbconn.Exec( - `INSERT INTO container (id, filename, current_size, max_size, sealed) - VALUES (1, $1, $2, $3, FALSE)`, - "ack_test_container.bin", - container.ContainerHdrLen, - container.GetContainerMaxSize(), - ); err != nil { - t.Fatalf("insert container row: %v", err) - } - tmpDir := t.TempDir() pathA := filepath.Join(tmpDir, "reuse-a.bin") pathB := filepath.Join(tmpDir, "reuse-b.bin") @@ -2675,7 +2666,7 @@ func setupCrossVersionSharedChunkScenario(t *testing.T) crossVersionSharedChunkS t.Fatalf("write second file: %v", err) } - writer := &commitAckWriter{} + writer := container.NewLocalWriterWithDirAndDB(tmpDir, container.GetContainerMaxSize(), dbconn) firstChunker := fixedBoundaryChunker{version: chunk.VersionV1SimpleRolling, boundary: 32} secondChunker := fixedBoundaryChunker{version: chunk.VersionV2FastCDC, boundary: 32} @@ -4213,6 +4204,49 @@ func TestValidateReusableLogicalFileForStoreRunsSemanticValidation(t *testing.T) } } +func TestValidateReusableLogicalFileForStoreRejectsZstdBoundFailure(t *testing.T) { + fileID, dbconn, workDir, blockID, _, _, _ := setupStoredBlockFixtureForReaderCorruption(t, blocks.CodecPlain, storagecompression.CompressionZstd) + + if _, err := dbconn.Exec(`UPDATE storage_blocks SET plaintext_size = $1 WHERE id = $2`, int64(1), blockID); err != nil { + t.Fatalf("update plaintext_size for semantic-reuse bound fixture: %v", err) + } + + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + t.Setenv("COLDKEEP_REUSE_SEMANTIC_VALIDATION", "always") + + err := validateReusableLogicalFileForStoreWithContext(ctx, dbconn, fileID, workDir) + if err == nil || !strings.Contains(err.Error(), "semantic reuse validation failed") { + t.Fatalf("expected semantic reuse bound failure, got: %v", err) + } + if !strings.Contains(err.Error(), "decompress codec=zstd") && !strings.Contains(err.Error(), "decompress codec=\"zstd\"") { + t.Fatalf("expected bounded decompression cause, got: %v", err) + } +} + +func TestMaximumReleasedPackedWriterOutputFitsDecompressionLimit(t *testing.T) { + t.Setenv("COLDKEEP_BLOCK_TARGET_SIZE_MB", "3") + t.Setenv("COLDKEEP_PACKED_BLOCK_SIZE_MIB", "") + + targetSize := packedBlockTargetSizeBytesFromEnv() + maxEntries := targetSize / int64(fastcdc.MinChunkSize) + const ( + ckblHeaderSize = int64(20) + ckblEntrySize = int64(24) + ) + maxEncodedSize := targetSize + ckblHeaderSize + maxEntries*ckblEntrySize + + if maxEntries != 96 { + t.Fatalf("released writer entry-bound drift: got=%d want=96", maxEntries) + } + if maxEncodedSize != 3_148_052 { + t.Fatalf("released writer encoded maximum drift: got=%d want=3148052", maxEncodedSize) + } + if maxEncodedSize >= storagecompression.MaxDecompressedBlockSize { + t.Fatalf("writer maximum must fit decompression limit: writer=%d limit=%d", maxEncodedSize, storagecompression.MaxDecompressedBlockSize) + } +} + func insertReusableTestLogicalFile(t *testing.T, dbconn *sql.DB, totalSize int64) int64 { t.Helper() diff --git a/internal/testutil/backendtest/backendtest.go b/internal/testutil/backendtest/backendtest.go new file mode 100644 index 00000000..ff100d36 --- /dev/null +++ b/internal/testutil/backendtest/backendtest.go @@ -0,0 +1,240 @@ +// Package backendtest provides isolated SQLite and optional PostgreSQL fixtures +// for package-level backend contract tests. It is test support, not a runtime +// database abstraction. +package backendtest + +import ( + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync/atomic" + "testing" + + "github.com/franchoy/coldkeep/internal/db" +) + +// PostgresMode controls how ForEach handles PostgreSQL. +type PostgresMode int + +const ( + // PostgresOptional runs PostgreSQL only when COLDKEEP_TEST_DB is set. + PostgresOptional PostgresMode = iota + // PostgresRequired fails when PostgreSQL was not explicitly configured. + PostgresRequired +) + +// SchemaMode controls whether a fixture is bootstrapped before its callback. +type SchemaMode int + +const ( + // CurrentSchema applies the current Coldkeep schema. + CurrentSchema SchemaMode = iota + // EmptySchema opens the database without applying schema. + EmptySchema +) + +// Options configures a dual-backend fixture. The zero value uses CurrentSchema +// and optional PostgreSQL. +type Options struct { + Postgres PostgresMode + Schema SchemaMode +} + +// Capabilities records current backend-specific behavior. It does not cause +// automatic skips; callers must assert their intended backend behavior. +type Capabilities struct { + SelectForUpdate bool + SkipLocked bool + Nowait bool + LiveGC bool +} + +// Backend is one isolated fixture passed to a contract-test callback. +type Backend struct { + Name string + Kind db.Backend + DB *sql.DB + Capabilities Capabilities +} + +var scratchCounter atomic.Uint64 + +// ForEach invokes fn for SQLite and for PostgreSQL when selected by options. +// PostgreSQL subtests are skipped only for optional mode with no +// COLDKEEP_TEST_DB setting; any configured setup failure is a test failure. +func ForEach(t *testing.T, options Options, fn func(t *testing.T, backend Backend)) { + t.Helper() + t.Run("sqlite", func(t *testing.T) { + fn(t, openSQLite(t, options.Schema)) + }) + t.Run("postgres", func(t *testing.T) { + if err := postgresSelectionError(options.Postgres, os.Getenv("COLDKEEP_TEST_DB") != ""); err != nil { + if errors.Is(err, errPostgresNotConfigured) && options.Postgres == PostgresOptional { + t.Skip("set COLDKEEP_TEST_DB=1 (with DB_* connection settings) to run PostgreSQL backend tests") + } + t.Fatal(err) + } + fn(t, openPostgres(t, options.Schema)) + }) +} + +func openSQLite(t *testing.T, schema SchemaMode) Backend { + t.Helper() + path := filepath.Join(t.TempDir(), "coldkeep.sqlite") + conn, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("open SQLite fixture: %v", err) + } + conn.SetMaxOpenConns(1) + t.Cleanup(func() { + if err := conn.Close(); err != nil { + t.Errorf("close SQLite fixture %q: %v", path, err) + } + }) + if err := db.ApplySQLiteSessionPragmas(conn); err != nil { + t.Fatalf("initialize SQLite fixture session: %v", err) + } + ensureSchema(t, conn, schema, "SQLite") + return backendFor("sqlite", conn) +} + +func openPostgres(t *testing.T, schema SchemaMode) Backend { + t.Helper() + adminName := getenvOrDefault("COLDKEEP_TEST_DB_MAINTENANCE", "postgres") + admin := openPostgresConnection(t, adminName, "admin") + t.Cleanup(func() { + if err := admin.Close(); err != nil { + t.Errorf("close PostgreSQL admin connection: %v", err) + } + }) + + name := scratchDatabaseName(t.Name()) + if _, err := admin.Exec("CREATE DATABASE " + quoteIdentifier(name)); err != nil { + t.Fatalf("create PostgreSQL scratch database %q: %v", name, err) + } + + var tested *sql.DB + // This cleanup is registered after the admin close cleanup, so it runs first. + // It remains valid if a later open or bootstrap step calls Fatal. + t.Cleanup(func() { + cleanupScratchDatabase(admin, tested, name, func(format string, args ...any) { + t.Errorf(format, args...) + }) + }) + tested = openPostgresConnection(t, name, "scratch database") + ensureSchema(t, tested, schema, "PostgreSQL") + return backendFor("postgres", tested) +} + +func ensureSchema(t *testing.T, conn *sql.DB, schema SchemaMode, backend string) { + t.Helper() + if schema == EmptySchema { + return + } + if schema != CurrentSchema { + t.Fatalf("unsupported %s fixture schema mode %d", backend, schema) + } + if err := db.EnsureSchema(conn); err != nil { + t.Fatalf("bootstrap %s fixture schema: %v", backend, err) + } +} + +func backendFor(name string, conn *sql.DB) Backend { + kind := db.BackendFromDB(conn) + locks := db.SupportsSelectForUpdate(conn) + return Backend{Name: name, Kind: kind, DB: conn, Capabilities: Capabilities{ + SelectForUpdate: locks, + SkipLocked: db.SupportsSelectForUpdateSkipLocked(conn), + Nowait: db.SupportsSelectForUpdateNowait(conn), + LiveGC: kind == db.BackendPostgres, + }} +} + +func openPostgresConnection(t *testing.T, databaseName, purpose string) *sql.DB { + t.Helper() + connString, err := db.BuildPostgresConnStringFromEnv(databaseName) + if err != nil { + t.Fatalf("build PostgreSQL %s connection string: %v", purpose, err) + } + conn, err := sql.Open("postgres", connString) + if err != nil { + t.Fatalf("open PostgreSQL %s connection: %v", purpose, err) + } + if err := conn.Ping(); err != nil { + _ = conn.Close() + t.Fatalf("ping PostgreSQL %s connection: %v", purpose, err) + } + return conn +} + +type sqlExecutor interface { + Exec(query string, args ...any) (sql.Result, error) +} +type dbCloser interface{ Close() error } + +func cleanupScratchDatabase(admin sqlExecutor, tested dbCloser, name string, report func(string, ...any)) { + if tested != nil { + if err := tested.Close(); err != nil { + report("close PostgreSQL scratch database %q: %v", name, err) + } + } + if _, err := admin.Exec(`SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = $1 AND pid <> pg_backend_pid()`, name); err != nil { + report("terminate sessions for PostgreSQL scratch database %q: %v", name, err) + } + if _, err := admin.Exec("DROP DATABASE IF EXISTS " + quoteIdentifier(name)); err != nil { + report("drop PostgreSQL scratch database %q: %v", name, err) + } +} + +var validIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + +func scratchDatabaseName(testName string) string { + const prefix = "coldkeep_bt_" + suffix := fmt.Sprintf("_%x_%x", os.Getpid(), scratchCounter.Add(1)) + name := strings.ToLower(testName) + name = strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' { + return r + } + return '_' + }, name) + name = strings.Trim(name, "_") + if name == "" { + name = "test" + } + maxBase := 63 - len(prefix) - len(suffix) + if len(name) > maxBase { + name = name[:maxBase] + } + return prefix + name + suffix +} + +func quoteIdentifier(name string) string { + if !validIdentifier.MatchString(name) { + panic("invalid generated PostgreSQL identifier: " + name) + } + return `"` + name + `"` +} + +var errPostgresNotConfigured = errors.New("COLDKEEP_TEST_DB is required for PostgreSQL backend tests") + +func postgresSelectionError(mode PostgresMode, configured bool) error { + if mode != PostgresOptional && mode != PostgresRequired { + return fmt.Errorf("unsupported PostgreSQL mode %d", mode) + } + if !configured { + return errPostgresNotConfigured + } + return nil +} + +func getenvOrDefault(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} diff --git a/internal/testutil/backendtest/backendtest_test.go b/internal/testutil/backendtest/backendtest_test.go new file mode 100644 index 00000000..504f4dd0 --- /dev/null +++ b/internal/testutil/backendtest/backendtest_test.go @@ -0,0 +1,158 @@ +package backendtest + +import ( + "database/sql" + "errors" + "os" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/db" +) + +func TestForEachSQLiteFixture(t *testing.T) { + var path string + var names []string + ForEach(t, Options{}, func(t *testing.T, backend Backend) { + names = append(names, backend.Name) + if backend.Name == "postgres" { + if backend.Kind != db.BackendPostgres || !backend.Capabilities.SelectForUpdate || !backend.Capabilities.LiveGC { + t.Fatalf("unexpected PostgreSQL backend: %+v", backend) + } + if _, err := backend.DB.Exec("CREATE TABLE fixture_probe_postgres (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + return + } + if backend.Name != "sqlite" || backend.Kind != db.BackendSQLite { + t.Fatalf("backend = %+v", backend) + } + if backend.Capabilities.SelectForUpdate || backend.Capabilities.LiveGC { + t.Fatalf("unexpected SQLite capabilities: %+v", backend.Capabilities) + } + if _, err := backend.DB.Exec("CREATE TABLE fixture_probe (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if _, err := backend.DB.Exec("INSERT INTO fixture_probe (id) VALUES (1)"); err != nil { + t.Fatal(err) + } + if err := backend.DB.QueryRow("PRAGMA database_list").Scan(new(int), new(string), &path); err != nil { + t.Fatal(err) + } + other, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer other.Close() + if err := db.ApplySQLiteSessionPragmas(other); err != nil { + t.Fatal(err) + } + var got int + if err := other.QueryRow("SELECT COUNT(*) FROM fixture_probe").Scan(&got); err != nil || got != 1 { + t.Fatalf("second SQLite connection got %d, %v", got, err) + } + }) + if path == "" { + t.Fatal("SQLite callback did not run") + } + wantNames := "sqlite" + if os.Getenv("COLDKEEP_TEST_DB") != "" { + wantNames += ",postgres" + } + if got := strings.Join(names, ","); got != wantNames { + t.Fatalf("selected callback names = %q, want %q", got, wantNames) + } +} + +func TestScratchDatabaseName(t *testing.T) { + a, b := scratchDatabaseName("Test Name/With Punctuation"), scratchDatabaseName("Test Name/With Punctuation") + if a == b || len(a) > 63 || !validIdentifier.MatchString(a) { + t.Fatalf("invalid scratch names %q %q", a, b) + } +} + +func TestSQLiteFixtureStateAndFilesAreIsolated(t *testing.T) { + var closedPath string + t.Run("first", func(t *testing.T) { + ForEach(t, Options{}, func(t *testing.T, backend Backend) { + if backend.Kind != db.BackendSQLite { + return + } + if err := backend.DB.QueryRow("SELECT COUNT(*) FROM fixture_isolation_probe").Scan(new(int)); err == nil { + t.Fatal("unexpected state from another SQLite fixture") + } + if _, err := backend.DB.Exec("CREATE TABLE fixture_isolation_probe (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if err := backend.DB.QueryRow("PRAGMA database_list").Scan(new(int), new(string), &closedPath); err != nil { + t.Fatal(err) + } + }) + }) + if closedPath == "" { + t.Fatal("SQLite fixture path was not observed") + } + if _, err := os.Stat(closedPath); !os.IsNotExist(err) { + t.Fatalf("SQLite fixture path remains after callback cleanup: %q, err=%v", closedPath, err) + } + ForEach(t, Options{}, func(t *testing.T, backend Backend) { + if backend.Kind != db.BackendSQLite { + return + } + if err := backend.DB.QueryRow("SELECT COUNT(*) FROM fixture_isolation_probe").Scan(new(int)); err == nil { + t.Fatal("SQLite state leaked into a separate harness invocation") + } + }) +} + +func TestPostgresSelectionPolicy(t *testing.T) { + if err := postgresSelectionError(PostgresOptional, false); !errors.Is(err, errPostgresNotConfigured) { + t.Fatalf("optional absent = %v", err) + } + if err := postgresSelectionError(PostgresRequired, false); !errors.Is(err, errPostgresNotConfigured) { + t.Fatalf("required absent = %v", err) + } + if err := postgresSelectionError(PostgresRequired, true); err != nil { + t.Fatal(err) + } + if err := postgresSelectionError(PostgresMode(99), true); err == nil { + t.Fatal("invalid mode succeeded") + } +} + +type recordedExec struct { + calls []string + failures map[string]error +} + +func (e *recordedExec) Exec(query string, _ ...any) (sql.Result, error) { + e.calls = append(e.calls, query) + for key, err := range e.failures { + if strings.Contains(query, key) { + return nil, err + } + } + return nil, nil +} + +type recordedCloser struct { + closed bool + err error +} + +func (c *recordedCloser) Close() error { c.closed = true; return c.err } + +func TestCleanupScratchDatabaseContinuesAfterErrors(t *testing.T) { + exec := &recordedExec{failures: map[string]error{"pg_terminate": errors.New("terminate"), "DROP DATABASE": errors.New("drop")}} + closer := &recordedCloser{err: errors.New("close")} + var reports []string + cleanupScratchDatabase(exec, closer, "coldkeep_bt_test_1", func(format string, args ...any) { + reports = append(reports, format) + }) + if !closer.closed || len(exec.calls) != 2 || !strings.Contains(exec.calls[1], "DROP DATABASE") { + t.Fatalf("cleanup order: closed=%v calls=%v", closer.closed, exec.calls) + } + if len(reports) != 3 || !strings.Contains(reports[0], "close") || !strings.Contains(reports[2], "drop") { + t.Fatalf("cleanup reports = %v", reports) + } +} diff --git a/internal/verify/verify_block_pipeline_test.go b/internal/verify/verify_block_pipeline_test.go index 71fa1ed0..39df4543 100644 --- a/internal/verify/verify_block_pipeline_test.go +++ b/internal/verify/verify_block_pipeline_test.go @@ -114,6 +114,51 @@ func TestVerifyStoredBlockCompressedZstdPasses(t *testing.T) { } } +func TestVerifyStoredBlockRejectsZstdOutputBeyondExpectedSizeAtDecompressStage(t *testing.T) { + logicalPayload := buildPipelineEncodedBytes(t, bytes.Repeat([]byte("bounded-verify-"), 64)) + zstdCompressor, err := storagecompression.NewZstdCompressor(3) + if err != nil { + t.Fatalf("NewZstdCompressor: %v", err) + } + compressedPayload, err := zstdCompressor.Compress(logicalPayload) + if err != nil { + t.Fatalf("Compress: %v", err) + } + + level := 3 + meta := BlockStorageMetadata{ + BlockID: 203, + ContainerID: 23, + ContainerOffset: 192, + ContainerName: "container_bounded_zstd.ck", + ContainerMaxSize: 1 << 20, + FormatVersion: 1, + Codec: "none", + PlaintextSize: int64(len(logicalPayload) - 1), + StoredSize: int64(len(compressedPayload)), + CompressionCodec: "zstd", + CompressionLevel: &level, + LogicalHash: blocks.HashLogical(logicalPayload), + CompressedHash: blocks.HashCompressed(compressedPayload), + PhysicalHash: blocks.HashPhysical(compressedPayload), + } + + _, err = VerifyStoredBlock(context.Background(), meta, staticContainerReader{payload: compressedPayload}) + if err == nil { + t.Fatal("expected bounded decompression failure") + } + var vf *VerifyFailure + if !errors.As(err, &vf) { + t.Fatalf("expected VerifyFailure, got: %v", err) + } + if vf.Stage != VerifyStageDecompress || vf.Category != verifyErrMetadataInvalid { + t.Fatalf("unexpected failure classification: stage=%q category=%q err=%v", vf.Stage, vf.Category, err) + } + if !errors.Is(err, storagecompression.ErrCompressionSizeMismatch) { + t.Fatalf("expected ErrCompressionSizeMismatch, got: %v", err) + } +} + func TestVerifyStoredBlockDecompressionControlledByPerBlockMetadata(t *testing.T) { logicalPayload := buildPipelineEncodedBytes(t, []byte("verify-metadata-controls-decompression")) zstd, err := storagecompression.NewZstdCompressor(3) diff --git a/internal/verify/verify_system.go b/internal/verify/verify_system.go index 238f5595..9efb8d25 100644 --- a/internal/verify/verify_system.go +++ b/internal/verify/verify_system.go @@ -3,6 +3,7 @@ package verify import ( "context" "database/sql" + "errors" "fmt" "log" @@ -11,6 +12,57 @@ import ( "github.com/franchoy/coldkeep/internal/utils_print" ) +type deepVerifyContainer struct { + ID int + Filename string + CurrentSize int64 + MaxSize int64 +} + +func loadDeepVerifyContainers(ctx context.Context, dbconn *sql.DB) ([]deepVerifyContainer, error) { + rows, err := dbconn.QueryContext(ctx, ` + SELECT ctr.id, ctr.filename, ctr.current_size, ctr.max_size + FROM container ctr + WHERE ctr.quarantine = FALSE + AND EXISTS ( + SELECT 1 + FROM storage_blocks sb + WHERE sb.container_id = ctr.id + ) + `) + if err != nil { + return nil, fmt.Errorf("failed to query deep-verify containers: %w", err) + } + + closeWithError := func(prior error) error { + if closeErr := rows.Close(); closeErr != nil { + closeErr = fmt.Errorf("failed to close deep-verify container rows: %w", closeErr) + if prior != nil { + return errors.Join(prior, closeErr) + } + return closeErr + } + return prior + } + + containers := make([]deepVerifyContainer, 0) + for rows.Next() { + var container deepVerifyContainer + if err := rows.Scan(&container.ID, &container.Filename, &container.CurrentSize, &container.MaxSize); err != nil { + return nil, closeWithError(fmt.Errorf("failed to scan container info: %w", err)) + } + containers = append(containers, container) + } + if err := rows.Err(); err != nil { + return nil, closeWithError(fmt.Errorf("row iteration failed for containers: %w", err)) + } + if err := closeWithError(nil); err != nil { + return nil, err + } + + return containers, nil +} + func printCounters(dbconn *sql.DB) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() @@ -236,56 +288,24 @@ func VerifySystemDeepWithContainersDir(dbconn *sql.DB, containersDir string) err } reader := FilesystemContainerReader{ContainersDir: containersDir} - // Count all non-quarantined containers that currently hold packed storage blocks. ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() - containerCount := 0 - containerCountErr := dbconn.QueryRowContext(ctx, ` - SELECT COUNT(*) - FROM container ctr - WHERE ctr.quarantine = FALSE - AND EXISTS ( - SELECT 1 - FROM storage_blocks sb - WHERE sb.container_id = ctr.id - ) - `).Scan(&containerCount) - if containerCountErr != nil { - log.Println(" ERROR ") - log.Printf("Failed to query deep-verify container count: %v", containerCountErr) - return fmt.Errorf("failed to query deep-verify container count: %w", containerCountErr) - } - - processedContainers := 0 - - containers, err := dbconn.QueryContext(ctx, ` - SELECT ctr.id, ctr.filename, ctr.current_size, ctr.max_size - FROM container ctr - WHERE ctr.quarantine = FALSE - AND EXISTS ( - SELECT 1 - FROM storage_blocks sb - WHERE sb.container_id = ctr.id - ) - `) + containers, err := loadDeepVerifyContainers(ctx, dbconn) if err != nil { log.Println(" ERROR ") log.Printf("Failed to query deep-verify containers: %v", err) - return fmt.Errorf("failed to query deep-verify containers: %w", err) + return err } - defer func() { _ = containers.Close() }() - for containers.Next() { + containerCount := len(containers) + processedContainers := 0 + for _, containerInfo := range containers { processedContainers++ - var containerID int - var filename string - var currentSize int64 - var maxSize int64 - if err := containers.Scan(&containerID, &filename, ¤tSize, &maxSize); err != nil { - appendDeepError(fmt.Errorf("failed to scan container info: %w", err)) - continue - } + containerID := containerInfo.ID + filename := containerInfo.Filename + currentSize := containerInfo.CurrentSize + maxSize := containerInfo.MaxSize log.Printf("Verifying container %d/%d: %s", processedContainers, containerCount, filename) fileSize := currentSize @@ -437,10 +457,6 @@ func VerifySystemDeepWithContainersDir(dbconn *sql.DB, containersDir string) err continue } } - if err := containers.Err(); err != nil { - appendDeepError(fmt.Errorf("row iteration failed for containers: %w", err)) - } - if len(errorList) > 0 { log.Println(" ERROR ") log.Printf("Found %d errors in deep verification of container files:", errorCount) diff --git a/internal/verify/verify_system_single_connection_test.go b/internal/verify/verify_system_single_connection_test.go new file mode 100644 index 00000000..558a91bf --- /dev/null +++ b/internal/verify/verify_system_single_connection_test.go @@ -0,0 +1,97 @@ +package verify_test + +import ( + "context" + "database/sql" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/blocks" + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/storage" + "github.com/franchoy/coldkeep/internal/verify" + _ "github.com/mattn/go-sqlite3" +) + +const verifySingleConnectionChild = "COLDKEEP_VERIFY_SINGLE_CONNECTION_CHILD" + +func TestVerifySystemDeepPackedSQLiteSingleConnection(t *testing.T) { + if os.Getenv(verifySingleConnectionChild) == "1" { + runVerifySystemDeepPackedSQLiteSingleConnection(t) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestVerifySystemDeepPackedSQLiteSingleConnection$", "-test.count=1") + env := make([]string, 0, len(os.Environ())+2) + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, verifySingleConnectionChild+"=") || + strings.HasPrefix(entry, "COLDKEEP_DB_OPERATION_TIMEOUT_MS=") { + continue + } + env = append(env, entry) + } + cmd.Env = append(env, + verifySingleConnectionChild+"=1", + "COLDKEEP_DB_OPERATION_TIMEOUT_MS=250", + ) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("single-connection deep verification child exceeded outer timeout: %v\n%s", ctx.Err(), output) + } + if err != nil { + t.Fatalf("single-connection deep verification child failed: %v\n%s", err, output) + } +} + +func runVerifySystemDeepPackedSQLiteSingleConnection(t *testing.T) { + t.Helper() + + root := t.TempDir() + dbconn, err := sql.Open("sqlite3", filepath.Join(root, "coldkeep.db")) + if err != nil { + t.Fatalf("open SQLite fixture: %v", err) + } + defer func() { _ = dbconn.Close() }() + dbconn.SetMaxOpenConns(1) + dbconn.SetMaxIdleConns(1) + if err := db.ApplySQLiteSessionPragmas(dbconn); err != nil { + t.Fatalf("apply SQLite pragmas: %v", err) + } + if err := db.EnsureSchema(dbconn); err != nil { + t.Fatalf("ensure current schema: %v", err) + } + + containersDir := filepath.Join(root, "containers") + writer := container.NewLocalWriterWithDirAndDB(containersDir, container.GetContainerMaxSize(), dbconn) + storageContext := storage.StorageContext{DB: dbconn, Writer: writer, ContainerDir: containersDir} + source := filepath.Join(root, "packed-source.bin") + if err := os.WriteFile(source, []byte("single-connection packed deep verification fixture"), 0o600); err != nil { + t.Fatalf("write source fixture: %v", err) + } + if _, err := storage.StoreFileWithStorageContextAndCodecResult(storageContext, source, blocks.CodecPlain); err != nil { + t.Fatalf("store packed fixture: %v", err) + } + if err := writer.FinalizeContainer(); err != nil { + t.Fatalf("finalize packed fixture container: %v", err) + } + + var storageBlockCount int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM storage_blocks`).Scan(&storageBlockCount); err != nil { + t.Fatalf("count packed storage blocks: %v", err) + } + if storageBlockCount < 1 { + t.Fatal("packed fixture must contain at least one storage_blocks row") + } + + if err := verify.VerifySystemDeepWithContainersDir(dbconn, containersDir); err != nil { + t.Fatalf("deep verification with one SQLite connection: %v", err) + } +} diff --git a/internal/verify/verify_system_test.go b/internal/verify/verify_system_test.go index 44a546b8..1a3bc089 100644 --- a/internal/verify/verify_system_test.go +++ b/internal/verify/verify_system_test.go @@ -1507,6 +1507,33 @@ func TestVerifyBlockPayloadsDetectsDecompressionFailureOnCompressedBlock(t *test } } +func TestVerifyBlockPayloadsRejectsZstdOutputBeyondExpectedSize(t *testing.T) { + dbconn := openVerifyTestDB(t) + defer func() { _ = dbconn.Close() }() + + containersDir := t.TempDir() + blockID, _ := seedVerifyCompressedPackedBlockFixture( + t, + dbconn, + containersDir, + [][]byte{bytes.Repeat([]byte("bounded-system-verify-"), 128)}, + blocks.CodecPlain, + storagecompression.CompressionZstd, + ) + + if _, err := dbconn.Exec(`UPDATE storage_blocks SET plaintext_size = $1 WHERE id = $2`, int64(1), blockID); err != nil { + t.Fatalf("update plaintext_size for bounded decompression fixture: %v", err) + } + + err := verifyBlockPayloads(dbconn, containersDir) + if err == nil || !strings.HasPrefix(err.Error(), "metadata_invalid:") { + t.Fatalf("expected metadata_invalid bounded decompression failure, got: %v", err) + } + if !strings.Contains(err.Error(), "decompress codec=zstd") { + t.Fatalf("expected decompress-stage diagnostic, got: %v", err) + } +} + func TestVerifyBlockPayloadsDetectsLogicalHashMismatchStageOnCompressedBlock(t *testing.T) { dbconn := openVerifyTestDB(t) defer func() { _ = dbconn.Close() }() @@ -1599,7 +1626,7 @@ func TestVerifyBlockPayloadsDetectsDecodedPayloadSizeMismatch(t *testing.T) { } err := verifyBlockPayloads(dbconn, containersDir) - if err == nil || !strings.HasPrefix(err.Error(), "metadata_invalid:") || !strings.Contains(err.Error(), "plaintext size mismatch") { + if err == nil || !strings.HasPrefix(err.Error(), "metadata_invalid:") || !strings.Contains(err.Error(), "decompression size mismatch") { t.Fatalf("expected decoded payload size mismatch error, got: %v", err) } } diff --git a/internal/version/version.go b/internal/version/version.go index 8be3f942..fc811c01 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -5,7 +5,7 @@ import "fmt" const ( Major = 1 Minor = 13 - Patch = 10 + Patch = 11 ) func String() string { diff --git a/internal/version/version_test.go b/internal/version/version_test.go index f8b90dee..35fa1eac 100644 --- a/internal/version/version_test.go +++ b/internal/version/version_test.go @@ -3,7 +3,7 @@ package version import "testing" func TestStringReturnsSemverFromConstants(t *testing.T) { - if got, want := String(), "1.13.10"; got != want { + if got, want := String(), "1.13.11"; got != want { t.Fatalf("String() mismatch: got=%q want=%q", got, want) } } diff --git a/scripts/audit_ci_enforcement.sh b/scripts/audit_ci_enforcement.sh index 6ccfa704..b56ecf5b 100755 --- a/scripts/audit_ci_enforcement.sh +++ b/scripts/audit_ci_enforcement.sh @@ -3,7 +3,7 @@ set -euo pipefail usage() { cat <<'EOF' -Usage: scripts/audit_ci_enforcement.sh [--repo owner/repo] [--local-only] [--remote-only] +Usage: scripts/audit_ci_enforcement.sh [--repo owner/repo] [--local-only] [--remote-only] [--paired-launcher FILE] Verifies the repo-side CI gate invariants and, when GitHub API access is available, audits the repository protection settings needed to make CI @@ -22,6 +22,7 @@ EOF REPO="" LOCAL_ONLY=0 REMOTE_ONLY=0 +PAIRED_LAUNCHER_FILE="" while [[ $# -gt 0 ]]; do case "$1" in @@ -41,6 +42,14 @@ while [[ $# -gt 0 ]]; do REMOTE_ONLY=1 shift ;; + --paired-launcher) + if [[ $# -lt 2 ]]; then + echo "[audit] ERROR: --paired-launcher requires a workflow path" >&2 + exit 2 + fi + PAIRED_LAUNCHER_FILE="$2" + shift 2 + ;; -h|--help) usage exit 0 @@ -62,7 +71,15 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd) WORKFLOW_FILE="${COLDKEEP_CI_WORKFLOW_FILE:-$REPO_ROOT/.github/workflows/ci.yml}" CODEQL_WORKFLOW_FILE="${COLDKEEP_CODEQL_WORKFLOW_FILE:-$REPO_ROOT/.github/workflows/codeql.yml}" +BENCHMARK_BASELINE_WORKFLOW_FILE="${COLDKEEP_BENCHMARK_BASELINE_WORKFLOW_FILE:-$REPO_ROOT/.github/workflows/benchmark-baseline.yml}" +BENCHMARK_GATE_FILE="${COLDKEEP_BENCHMARK_GATE_FILE:-$REPO_ROOT/scripts/benchmark_gate.py}" +TIMING_VALIDATOR_FILE="${COLDKEEP_TIMING_VALIDATOR_FILE:-$REPO_ROOT/scripts/validate_regression_thresholds.py}" VALIDATION_MATRIX_FILE="${COLDKEEP_VALIDATION_MATRIX_FILE:-$REPO_ROOT/VALIDATION_MATRIX.md}" +PAIRED_REFERENCE_MANIFEST_FILE="${COLDKEEP_PAIRED_REFERENCE_MANIFEST_FILE:-$REPO_ROOT/benchmarks/paired/reference-v1.13.json}" +PAIRED_THRESHOLD_POLICY_FILE="${COLDKEEP_PAIRED_THRESHOLD_POLICY_FILE:-$REPO_ROOT/benchmarks/paired/threshold-policy-v1.13.json}" +NATIVE_UNIX_TEST_FILE="${COLDKEEP_NATIVE_UNIX_TEST_FILE:-$REPO_ROOT/internal/coordination/native_lock_unix_test.go}" +NATIVE_WINDOWS_TEST_FILE="${COLDKEEP_NATIVE_WINDOWS_TEST_FILE:-$REPO_ROOT/internal/coordination/native_lock_windows_test.go}" +COORDINATOR_NATIVE_TEST_FILE="${COLDKEEP_COORDINATOR_NATIVE_TEST_FILE:-$REPO_ROOT/internal/coordination/coordinator_native_test.go}" require_pattern() { local file="$1" @@ -105,6 +122,22 @@ extract_job_block() { ' "$WORKFLOW_FILE" } +extract_job_block_from_file() { + local file="$1" + local job_name="$2" + awk -v job_name="$job_name" ' + $0 ~ ("^ " job_name ":$") { + in_job = 1 + } + in_job && $0 ~ "^ [A-Za-z0-9_-]+:$" && $0 !~ ("^ " job_name ":$") { + exit + } + in_job { + print + } + ' "$file" +} + extract_step_block_from_content() { local content="$1" local step_name="$2" @@ -121,20 +154,524 @@ extract_step_block_from_content() { ' <<<"$content" } +check_paired_launcher_output_ownership() { + local file="$1" + + python3 - "$file" <<'PY' +import pathlib +import posixpath +import re +import shlex +import sys + +source = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +assignments = { + match.group(1): match.group(3) + for match in re.finditer( + r"(?m)^[ \t]*([A-Za-z_][A-Za-z0-9_]*)=([\"'])(.*?)\2[ \t]*$", source + ) +} +errors: list[str] = [] + + +def error(message: str) -> None: + if message not in errors: + errors.append(message) + + +def expand(value: str) -> str: + value = value.strip().strip("\"'") + for _ in range(12): + previous = value + for name, replacement in assignments.items(): + value = value.replace("${" + name + "}", replacement) + value = re.sub(r"\$" + re.escape(name) + r"\b", replacement, value) + if value == previous: + break + value = value.replace("${GITHUB_WORKSPACE}", "@workspace") + value = re.sub(r"\$GITHUB_WORKSPACE\b", "@workspace", value) + value = value.replace("${RUNNER_TEMP}", "@runner_temp") + value = re.sub(r"\$RUNNER_TEMP\b", "@runner_temp", value) + value = value.rstrip("/") or "/" + if not value.startswith(("@workspace", "@runner_temp", "/")): + value = "@workspace/" + value.lstrip("./") + return re.sub(r"/{2,}", "/", value) + + +def shell_tokens(line: str) -> list[str]: + try: + return shlex.split(line.rstrip(" \\")) + except ValueError: + return [] + + +def command_blocks() -> list[dict[str, object]]: + blocks: list[dict[str, object]] = [] + pattern = re.compile(r"paired_benchmark_gate\.py[ \t]+(sample|decision)\b") + for match in pattern.finditer(source): + step_start = source.rfind("\n - name:", 0, match.start()) + if step_start < 0: + step_start = source.rfind("\n steps:", 0, match.start()) + if step_start < 0: + step_start = 0 + step_end = source.find("\n - name:", match.end()) + if step_end < 0: + step_end = len(source) + tail = source[match.end():step_end] + output = re.search( + r"--output-dir[ \t]+(?:\"([^\"\n]+)\"|'([^'\n]+)'|([^\s\\]+))", tail + ) + if output is None: + error(f"{match.group(1)} command must pass --output-dir") + continue + token = next(group for group in output.groups() if group is not None) + blocks.append( + { + "kind": match.group(1), + "start": match.start(), + "end": step_end, + "step_start": step_start, + "token": token, + "path": expand(token), + } + ) + return blocks + + +def creation_targets(text: str) -> tuple[set[str], set[str]]: + created: set[str] = set() + populated: set[str] = set() + for raw_line in text.splitlines(): + line = raw_line.strip() + tokens = shell_tokens(line) + if not tokens: + continue + command = tokens[0] + operands = [token for token in tokens[1:] if not token.startswith("-")] + if command in {"mkdir", "touch"}: + created.update(expand(token) for token in operands) + elif command == "install" and "-d" in tokens: + created.update(expand(token) for token in operands) + elif command in {"cp", "mv", "rsync", "git"} and operands: + populated.add(expand(operands[-1])) + elif command in {"tar", "unzip"}: + for option in ("-C", "--directory", "-d"): + if option in tokens and tokens.index(option) + 1 < len(tokens): + populated.add(expand(tokens[tokens.index(option) + 1])) + elif command == "ln" and "-s" in tokens and operands: + populated.add(expand(operands[-1])) + elif command == "rm": + populated.update(expand(token) for token in operands) + return created, populated + + +blocks = command_blocks() +sample_paths: list[str] = [] +all_output_paths = {str(block["path"]) for block in blocks} + +for block in blocks: + kind = str(block["kind"]) + output_path = str(block["path"]) + token = str(block["token"]) + start = int(block["start"]) + step_start = int(block["step_start"]) + prefix = source[:start] + step_prefix = source[step_start:start] + + if kind == "sample": + sample_paths.append(output_path) + + if output_path in {".", "/", "@workspace", "@runner_temp"}: + error(f"{kind} output must be a nonexistent child below a contained parent") + if not output_path.startswith(("@workspace/", "@runner_temp/")): + error(f"{kind} output must remain below a permitted contained parent") + if any(part == ".." for part in output_path.split("/")): + error(f"{kind} output must not use traversal") + if output_path.startswith(("@workspace/candidate", "@workspace/reference")): + error(f"{kind} output must not reuse a repository checkout path") + + parent = posixpath.dirname(output_path) + created, populated = creation_targets(prefix) + if parent not in created: + error(f"{kind} output parent must exist before harness invocation") + if output_path in created: + error(f"{kind} output must not be created before harness invocation") + if output_path in populated: + error(f"{kind} output must not be populated, checked out, extracted, or recreated") + + assertions = re.finditer( + r"test[ \t]+![ \t]+-e[ \t]+(?:\"([^\"\n]+)\"|'([^'\n]+)'|([^\s\\]+))", + step_prefix, + ) + asserted_paths = { + expand(next(group for group in assertion.groups() if group is not None)) + for assertion in assertions + } + if output_path not in asserted_paths: + error(f"{kind} output requires an exact nonexistence assertion before invocation") + + for checkout in re.finditer(r"uses:[ \t]*actions/checkout@[^\n]+", prefix): + checkout_end = source.find("\n - name:", checkout.end()) + if checkout_end < 0 or checkout_end > start: + checkout_end = start + checkout_block = source[checkout.start():checkout_end] + path_match = re.search(r"(?m)^[ \t]*path:[ \t]*([^\n#]+)", checkout_block) + if path_match and expand(path_match.group(1)) == output_path: + error(f"{kind} output must not be an actions/checkout destination") + + upload = re.search(r"uses:[ \t]*actions/upload-artifact@[^\n]+", source[int(block["end"]):]) + if upload is None: + error(f"{kind} output must be uploaded after harness execution") + continue + upload_start = int(block["end"]) + upload.start() + upload_end = source.find("\n - name:", upload_start + 1) + if upload_end < 0: + upload_end = len(source) + upload_block = source[upload_start:upload_end] + upload_path = re.search(r"(?m)^[ \t]*path:[ \t]*([^\n#]+)", upload_block) + if upload_path is None or expand(upload_path.group(1)) != output_path: + error(f"{kind} upload path must equal the harness-owned output path") + +if "${{ matrix.profile }}" in source and any( + "${{ matrix.profile }}" not in path for path in sample_paths +): + error("sample output must be distinct for every matrix profile") +if len(sample_paths) != len(set(sample_paths)): + error("sample profiles must not reuse an output directory") +if len(all_output_paths) != len(blocks): + error("sample and decision commands must use distinct output directories") + +generated: set[str] = set() +for name, value in assignments.items(): + if re.search(r"mktemp|openssl[ \t]+rand|uuidgen|\$RANDOM", value): + generated.add(name) +for name in generated: + mask = re.search(r"::add-mask::[^\n]*(?:\$\{" + re.escape(name) + r"\}|\$" + re.escape(name) + r"\b)", source) + prints = [ + match + for match in re.finditer( + r"(?m)^[ \t]*(?:echo|printf)[^\n]*(?:\$\{" + re.escape(name) + r"\}|\$" + re.escape(name) + r"\b)", + source, + ) + if "::add-mask::" not in match.group(0) + ] + if prints and (mask is None or prints[0].start() < mask.start()): + error("generated dynamic paths and identifiers must be masked before printing") + +if re.search(r"--mode[ \t]+production\b", source): + error("paired launcher must remain diagnostic-only") +if re.search(r"reference-v1\.13\.json|threshold-policy-v1\.13\.json|--manifest|--threshold", source): + error("paired launcher must not create manifest or threshold authority") + +if errors: + for message in errors: + print(f"[audit] ERROR: {message}", file=sys.stderr) + raise SystemExit(1) +print("[audit] ok: paired launcher preserves harness-owned nonexistent output children") +print("[audit] ok: paired launcher uploads the exact sample and decision outputs") +print("[audit] ok: static GitHub runtime aliases are permitted metadata") +PY +} + +check_paired_launcher() { + local file="$1" + local check_status=0 + local prohibited_env='^[[:space:]]+(COLDKEEP_KEY|COLDKEEP_AES_GCM_FIXTURE_HEX|DB_HOST|DB_PORT|DB_USER|DB_PASSWORD|DB_NAME|DB_SSLMODE|POSTGRES_USER|POSTGRES_PASSWORD|POSTGRES_DB):' + + echo "[audit] checking paired diagnostic launcher confidentiality and lifecycle" + if [[ ! -f "$file" || -L "$file" ]]; then + echo "[audit] ERROR: paired launcher must be a regular non-symlink file" >&2 + return 1 + fi + require_pattern "$file" '^ timeout-minutes: 45$' 'paired launcher outer timeout is 45 minutes' || check_status=1 + require_pattern "$file" 'ci-paired-w1-v2' 'paired launcher selects workers=1 v2 fixture' || check_status=1 + require_pattern "$file" 'ci-paired-w4-v2' 'paired launcher selects workers=4 v2 fixture' || check_status=1 + require_pattern "$file" '--pairs 10' 'paired launcher retains ten diagnostic pairs' || check_status=1 + require_pattern "$file" '--command-timeout-seconds 600' 'paired launcher retains 600-second command safety timeout' || check_status=1 + require_pattern "$file" 'paired_benchmark_gate\.py sample' 'paired launcher uses strict sampler' || check_status=1 + require_pattern "$file" 'paired_benchmark_gate\.py decision' 'paired launcher writes a matrix decision' || check_status=1 + require_pattern "$file" 'if: \$\{\{ always\(\) \}\}' 'paired launcher always uploads finalized evidence' || check_status=1 + require_pattern "$file" 'set \+x' 'paired launcher disables shell tracing before sensitive setup' || check_status=1 + for variable in GITHUB_WORKSPACE RUNNER_TEMP HOME; do + require_pattern "$file" "::add-mask::.*\\\$${variable}" "paired launcher masks ${variable}" || check_status=1 + done + if grep -Eq '^ services:|^ services:' "$file"; then + echo "[audit] ERROR: paired launcher must provision its isolated container after masking" >&2 + check_status=1 + else + echo "[audit] ok: paired launcher does not use pre-step service provisioning" + fi + if grep -Eq "$prohibited_env" "$file"; then + echo "[audit] ERROR: paired launcher exposes prohibited values through YAML env" >&2 + check_status=1 + else + echo "[audit] ok: paired launcher has no prohibited YAML env exposure" + fi + if grep -Eq 'GITHUB_ENV|set -x' "$file"; then + echo "[audit] ERROR: paired launcher persists or traces sensitive runtime values" >&2 + check_status=1 + else + echo "[audit] ok: paired launcher neither persists nor traces sensitive runtime values" + fi + if grep -Eq 'echo.*(GITHUB_WORKSPACE|RUNNER_TEMP|COLDKEEP_KEY|DB_PASSWORD|DB_NAME)' "$file" \ + && ! grep -Eq '::add-mask::.*(GITHUB_WORKSPACE|RUNNER_TEMP)' "$file"; then + echo "[audit] ERROR: paired launcher may print a prohibited path or runtime value" >&2 + check_status=1 + fi + check_paired_launcher_output_ownership "$file" || check_status=1 + return "$check_status" +} + check_local_workflow() { local check_status=0 local adversarial_block="" + local benchmark_authorize_block="" + local benchmark_calibration_block="" local deterministic_g6_block="" + local benchmark_permissions_block="" + local benchmark_sample_block="" + local cache_disabled_count=0 + local checkout_count=0 + local credential_disabled_count=0 local quality_block="" local quality_checkout_block="" local validator_test_block="" local validator_real_block="" + local benchmark_integrity_block="" + local benchmark_timing_block="" + local correctness_matrix_block="" + local postgres_internal_contracts_block="" + local setup_go_count=0 + local trusted_checkout_count=0 local upload_v5_count=0 local upload_v6_count=0 local upload_v7_count=0 echo "[audit] checking local workflow invariants" require_pattern "$WORKFLOW_FILE" 'name: CI' 'CI workflow file' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^name: Benchmark Gate Calibration and Baseline Capture$' 'manual benchmark calibration workflow' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^ workflow_dispatch:$' 'benchmark calibration is manually dispatched' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^ contents: read$' 'benchmark calibration has read-only repository permission' || check_status=1 + benchmark_authorize_block="$(extract_job_block_from_file "$BENCHMARK_BASELINE_WORKFLOW_FILE" authorize)" + benchmark_sample_block="$(extract_job_block_from_file "$BENCHMARK_BASELINE_WORKFLOW_FILE" sample)" + benchmark_calibration_block="$(extract_job_block_from_file "$BENCHMARK_BASELINE_WORKFLOW_FILE" calibration)" + benchmark_permissions_block="$(awk ' + /^permissions:$/ { in_permissions = 1 } + in_permissions && /^env:$/ { exit } + in_permissions { print } + ' "$BENCHMARK_BASELINE_WORKFLOW_FILE")" + if [[ "$benchmark_permissions_block" != $'permissions:\n contents: read' ]]; then + echo "[audit] ERROR: benchmark workflow permissions must be exactly contents read" >&2 + check_status=1 + else + echo "[audit] ok: benchmark workflow permissions are exactly contents read" + fi + require_content_pattern "$benchmark_authorize_block" 'if \[\[ "\$\{TRUSTED_REF\}" != "refs/heads/main" \]\]; then' 'benchmark authorization is fail-closed on refs/heads/main' || check_status=1 + require_content_pattern "$benchmark_authorize_block" 'if ! \[\[ "\$\{SOURCE_SHA\}" =~ \^\[0-9a-f\]\{40\}\$ \]\]; then' 'benchmark source_sha uses strict lowercase full-SHA validation' || check_status=1 + require_content_pattern "$benchmark_authorize_block" 'if \[\[ "\$\{SOURCE_SHA\}" != "\$\{TRUSTED_SHA\}" \]\]; then' 'benchmark source_sha must equal trusted github.sha' || check_status=1 + require_content_pattern "$benchmark_sample_block" '^ needs: authorize$' 'benchmark sample job depends on trusted-source authorization' || check_status=1 + if grep -Eq 'ref:.*inputs\.source_sha' "$BENCHMARK_BASELINE_WORKFLOW_FILE"; then + echo "[audit] ERROR: benchmark checkout cannot use inputs.source_sha" >&2 + check_status=1 + else + echo "[audit] ok: benchmark checkout does not use inputs.source_sha" + fi + checkout_count="$(grep -Ec 'uses: actions/checkout@' "$BENCHMARK_BASELINE_WORKFLOW_FILE")" + trusted_checkout_count="$(grep -Fc "ref: \${{ github.sha }}" "$BENCHMARK_BASELINE_WORKFLOW_FILE")" + credential_disabled_count="$(grep -Ec '^\s+persist-credentials: false$' "$BENCHMARK_BASELINE_WORKFLOW_FILE")" + if [[ "$checkout_count" -eq 0 || "$trusted_checkout_count" -ne "$checkout_count" ]]; then + echo "[audit] ERROR: benchmark checkouts must use trusted github.sha" >&2 + check_status=1 + else + echo "[audit] ok: benchmark checkouts use trusted github.sha" + fi + if [[ "$credential_disabled_count" -ne "$checkout_count" ]]; then + echo "[audit] ERROR: benchmark checkouts must disable persisted credentials" >&2 + check_status=1 + else + echo "[audit] ok: benchmark checkouts disable persisted credentials" + fi + setup_go_count="$(grep -Ec 'uses: actions/setup-go@' "$BENCHMARK_BASELINE_WORKFLOW_FILE")" + cache_disabled_count="$(grep -Ec '^\s+cache: false$' "$BENCHMARK_BASELINE_WORKFLOW_FILE")" + if [[ "$setup_go_count" -eq 0 || "$cache_disabled_count" -ne "$setup_go_count" ]] \ + || grep -Eq '^\s+cache: true$|uses: actions/cache@' "$BENCHMARK_BASELINE_WORKFLOW_FILE"; then + echo "[audit] ERROR: benchmark setup-go caching must be disabled" >&2 + check_status=1 + else + echo "[audit] ok: benchmark setup-go caching is disabled" + fi + require_content_pattern "$benchmark_sample_block" '^\s+python3 scripts/benchmark_gate\.py sample \\$' 'benchmark sample harness runs from trusted checkout' || check_status=1 + require_content_pattern "$benchmark_calibration_block" '^\s+python3 scripts/benchmark_gate\.py calibrate \\$' 'benchmark calibration harness runs from trusted checkout' || check_status=1 + require_content_pattern "$benchmark_calibration_block" 'path: \$\{\{ runner\.temp \}\}/benchmark-calibration-input' 'benchmark calibration artifacts use runner.temp' || check_status=1 + require_content_pattern "$benchmark_calibration_block" 'actual = report\.get\("provenance", \{\}\)\.get\("source_commit"\)' 'benchmark calibration reads artifact source provenance' || check_status=1 + require_content_pattern "$benchmark_calibration_block" 'if actual != expected:' 'benchmark calibration requires artifact provenance to match github.sha' || check_status=1 + if grep -Eq 'continue-on-error|\|\| true|^\s+set \+e$' <<<"$benchmark_authorize_block"; then + echo "[audit] ERROR: benchmark source validation must not use broad failure suppression" >&2 + check_status=1 + else + echo "[audit] ok: benchmark source validation has no broad failure suppression" + fi + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" 'runs-on: ubuntu-24\.04' 'benchmark calibration pins the runner family' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" "go-version: '1\\.25\\.12'" 'benchmark calibration pins the Go patch' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" 'postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20' 'benchmark calibration pins the PostgreSQL image digest' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+compression: \[none, zstd\]$' 'benchmark calibration fixes compression profiles' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+workers: \[1, 4\]$' 'benchmark calibration fixes worker profiles' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+replicate: \[1, 2\]$' 'benchmark calibration uses two independent matrix jobs' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+sample_count=10$' 'benchmark calibration fixes ten measured samples' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+sample_count=5$' 'benchmark capture fixes five measured samples' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" 'python3 scripts/benchmark_gate\.py sample' 'benchmark calibration uses the strict sampler' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+--dataset ci-stable-v1 \\$' 'benchmark calibration fixes the fixture identity' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^\s+--warmups 1 \\$' 'benchmark calibration fixes one excluded warmup' || check_status=1 + require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" 'python3 scripts/benchmark_gate\.py calibrate' 'benchmark calibration evaluates the fixed matrix' || check_status=1 + require_pattern "$WORKFLOW_FILE" '^ benchmark-integrity:$' 'hard benchmark integrity job family' || check_status=1 + require_pattern "$WORKFLOW_FILE" '^ benchmark-timing-advisory:$' 'hosted benchmark timing advisory job family' || check_status=1 + benchmark_integrity_block="$(extract_job_block benchmark-integrity)" + benchmark_timing_block="$(extract_job_block benchmark-timing-advisory)" + require_content_pattern "$benchmark_integrity_block" 'ci-paired-w1-v2' 'integrity matrix selects the bounded workers=1 fixture' || check_status=1 + require_content_pattern "$benchmark_integrity_block" 'ci-paired-w4-v2' 'integrity matrix selects the bounded workers=4 fixture' || check_status=1 + require_content_pattern "$benchmark_integrity_block" 'python3 scripts/benchmark_gate\.py integrity' 'integrity matrix uses the hard candidate-only interface' || check_status=1 + require_content_pattern "$benchmark_integrity_block" '--command-timeout-seconds 600' 'integrity matrix fixes the 600-second command timeout' || check_status=1 + require_content_pattern "$benchmark_integrity_block" "go-version: '1\.25\.12'" 'integrity matrix pins the Go patch version' || check_status=1 + require_content_pattern "$benchmark_integrity_block" 'postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20' 'integrity matrix pins PostgreSQL by digest' || check_status=1 + require_content_pattern "$benchmark_integrity_block" 'if-no-files-found: error' 'integrity artifact rejects missing evidence' || check_status=1 + require_content_pattern "$benchmark_integrity_block" 'if: \$\{\{ always\(\) \}\}' 'integrity artifact finalization and upload always run' || check_status=1 + require_content_pattern "$benchmark_integrity_block" 'sha256sum --check checksums\.sha256' 'integrity artifact checksum inventory is verified' || check_status=1 + require_content_pattern "$benchmark_timing_block" '^\s+--dataset small \\$' 'timing advisory retains the historical small fixture' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'scripts/validate_regression_thresholds\.py check' 'timing advisory retains the historical comparator' || check_status=1 + require_content_pattern "$benchmark_timing_block" '--policy hosted-advisory' 'timing comparator has informational authority' || check_status=1 + require_content_pattern "$benchmark_timing_block" '^\s+set \+e$' 'timing advisory disables errexit only for comparator evaluation' || check_status=1 + require_content_pattern "$benchmark_timing_block" '^\s+set -e$' 'timing advisory restores errexit immediately after comparator evaluation' || check_status=1 + if ! grep -A1 -E '^\s+comparator_exit=\$\?$' <<<"$benchmark_timing_block" | grep -Eq '^\s+set -e$'; then + echo "[audit] ERROR: timing advisory must restore errexit immediately after capturing comparator exit" >&2 + check_status=1 + else + echo "[audit] ok: timing advisory restores errexit immediately after capturing comparator exit" + fi + require_content_pattern "$benchmark_timing_block" '\[\[ -s "\$\{report\}" \]\]' 'timing advisory requires a machine-readable report' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'verify-advisory-exit' 'timing advisory verifies exact classification and exit code' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'comparator_exit=\$\?' 'timing advisory captures the comparator exit exactly' || check_status=1 + require_content_pattern "$benchmark_timing_block" '^\s+0\|10\|11\|12\)$' 'timing advisory narrowly accepts valid informational exit codes' || check_status=1 + require_content_pattern "$benchmark_timing_block" '^\s+2\)$' 'timing advisory preserves evaluator exit code 2 as failure' || check_status=1 + if ! grep -A1 -E '^\s+2\)$' <<<"$benchmark_timing_block" | grep -Eq '^\s+exit 2$'; then + echo "[audit] ERROR: timing advisory must return failure for evaluator exit code 2" >&2 + check_status=1 + else + echo "[audit] ok: timing advisory returns failure for evaluator exit code 2" + fi + require_content_pattern "$benchmark_timing_block" 'GITHUB_STEP_SUMMARY' 'timing advisory publishes its classification summary' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'historical_v1\.9|benchmarks/v1\.9/baselines' 'timing advisory alone cites historical v1.9 baselines' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'if-no-files-found: error' 'timing artifact rejects missing evidence' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'if: \$\{\{ always\(\) \}\}' 'timing artifact upload always runs' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'actual_inventory=.*find .*checksums\.sha256' 'timing artifact inventory is enumerated exhaustively' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'benchmark\.json\\ntiming-advisory\.json' 'timing artifact inventory is restricted to the report and observation' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'sha256sum benchmark\.json timing-advisory\.json > checksums\.sha256' 'timing artifact creates exhaustive checksums' || check_status=1 + require_content_pattern "$benchmark_timing_block" 'sha256sum --check checksums\.sha256' 'timing artifact verifies checksums' || check_status=1 + checksum_line="$(grep -nEm1 'sha256sum --check checksums\.sha256' <<<"$benchmark_timing_block" | cut -d: -f1 || true)" + evaluator_failure_line="$(grep -nEm1 '^\s+2\)$' <<<"$benchmark_timing_block" | cut -d: -f1 || true)" + if [[ -z "$checksum_line" || -z "$evaluator_failure_line" || "$checksum_line" -ge "$evaluator_failure_line" ]]; then + echo "[audit] ERROR: timing checksums must be finalized before evaluator code 2 fails the job" >&2 + check_status=1 + else + echo "[audit] ok: timing checksums are finalized before evaluator code 2 fails the job" + fi + require_pattern "$TIMING_VALIDATOR_FILE" '^TIMING_ROW_OPTIONAL_FIELDS = \{"diagnostic_final_state"\}$' 'historical timing treats diagnostic final state as optional' || check_status=1 + require_pattern "$TIMING_VALIDATOR_FILE" 'not legacy and "diagnostic_final_state" in row' 'optional timing diagnostic final state is validated when present' || check_status=1 + require_pattern "$TIMING_VALIDATOR_FILE" '"BENCHMARK_TIMING_EVALUATION_FAILURE": 2' 'timing evaluator failure maps exactly to exit code 2' || check_status=1 + require_pattern "$TIMING_VALIDATOR_FILE" '^EXECUTION_STATS_OMITTABLE_ZERO_FIELDS = \{' 'timing validator models Go omitempty counters explicitly' || check_status=1 + omitempty_block="$(awk ' + /^EXECUTION_STATS_OMITTABLE_ZERO_FIELDS = \{/ { in_block = 1 } + in_block { print } + in_block && /^\}$/ { exit } + ' "$TIMING_VALIDATOR_FILE")" + for field in container_append_count fsync_count container_open_count container_close_count snapshot_metadata_write_count; do + require_content_pattern "$omitempty_block" "\"${field}\"" "timing validator models Go omitempty field ${field}" || check_status=1 + done + if grep -Eq 'benchmark_contract\.hard_final_state' "$TIMING_VALIDATOR_FILE"; then + echo "[audit] ERROR: historical timing advisory must not require hard diagnostic final state" >&2 + check_status=1 + else + echo "[audit] ok: historical timing advisory does not require hard diagnostic final state" + fi + require_pattern "$BENCHMARK_GATE_FILE" '^RAW_ROW_FIELDS = \{' 'hard integrity keeps an explicit raw row contract' || check_status=1 + require_pattern "$BENCHMARK_GATE_FILE" '^def hard_final_state\(' 'hard integrity still requires diagnostic final-state authority' || check_status=1 + require_pattern "$BENCHMARK_GATE_FILE" '^INTEGRITY_SAMPLE_COUNT = 2$' 'integrity interface fixes two candidate samples' || check_status=1 + require_pattern "$BENCHMARK_GATE_FILE" '^INTEGRITY_COMMAND_TIMEOUT_SECONDS = 600$' 'integrity interface fixes the command ceiling' || check_status=1 + require_pattern "$BENCHMARK_GATE_FILE" '"warmup_count": 0' 'integrity interface has no warmup invocation' || check_status=1 + require_pattern "$BENCHMARK_GATE_FILE" '"performance_authority": False' 'integrity evidence is ineligible for performance authority' || check_status=1 + workflow_baseline_count=$(grep -c 'benchmarks/v1\.9/baselines/' "$WORKFLOW_FILE" || true) + timing_baseline_count=$(grep -c 'benchmarks/v1\.9/baselines/' <<<"$benchmark_timing_block" || true) + if [[ "$workflow_baseline_count" -ne "$timing_baseline_count" ]]; then + echo "[audit] ERROR: historical baselines appear outside timing advisory policy" >&2 + check_status=1 + else + echo "[audit] ok: historical baselines appear only under timing advisory policy" + fi + if grep -Eq 'continue-on-error|\|\| true' <<<"$benchmark_integrity_block$benchmark_timing_block"; then + echo "[audit] ERROR: benchmark integrity or advisory execution uses broad failure suppression" >&2 + check_status=1 + else + echo "[audit] ok: benchmark execution has no broad failure suppression" + fi + for profile in none-w1 none-w4 zstd-w1 zstd-w4; do + if [[ "$(grep -c -- "profile: $profile" <<<"$benchmark_integrity_block")" -ne 1 ]]; then + echo "[audit] ERROR: integrity matrix must contain profile $profile exactly once" >&2 + check_status=1 + fi + if [[ "$(grep -c -- "profile: $profile" <<<"$benchmark_timing_block")" -ne 1 ]]; then + echo "[audit] ERROR: timing advisory matrix must contain profile $profile exactly once" >&2 + check_status=1 + fi + done + if [[ "$(grep -c -- 'dataset: ci-paired-w1-v2' <<<"$benchmark_integrity_block")" -ne 2 ]]; then + echo "[audit] ERROR: integrity matrix must bind exactly two profiles to the bounded workers=1 fixture" >&2 + check_status=1 + fi + if [[ "$(grep -c -- 'dataset: ci-paired-w4-v2' <<<"$benchmark_integrity_block")" -ne 2 ]]; then + echo "[audit] ERROR: integrity matrix must bind exactly two profiles to the bounded workers=4 fixture" >&2 + check_status=1 + fi + if [[ -e "$PAIRED_REFERENCE_MANIFEST_FILE" ]]; then + echo "[audit] ERROR: paired reference manifest exists before governance authorization" >&2 + check_status=1 + else + echo "[audit] ok: no paired reference manifest exists" + fi + if [[ -e "$PAIRED_THRESHOLD_POLICY_FILE" ]]; then + echo "[audit] ERROR: paired threshold policy exists before threshold authorization" >&2 + check_status=1 + else + echo "[audit] ok: no paired threshold policy exists" + fi + if grep -Eq '^ (push|pull_request|merge_group|schedule):' "$BENCHMARK_BASELINE_WORKFLOW_FILE"; then + echo "[audit] ERROR: benchmark calibration workflow must remain manual-only" >&2 + check_status=1 + else + echo "[audit] ok: benchmark calibration workflow has no automatic trigger" + fi + if grep -Eq '^\s+(contents|actions|checks|issues|pull-requests):\s*write|^\s*write-all\s*$' "$BENCHMARK_BASELINE_WORKFLOW_FILE"; then + echo "[audit] ERROR: benchmark calibration workflow must not receive write permission" >&2 + check_status=1 + else + echo "[audit] ok: benchmark calibration workflow has no repository write permission" + fi + if grep -Eqi 'git (commit|push)|gh (pr|release)|create-pull-request' "$BENCHMARK_BASELINE_WORKFLOW_FILE"; then + echo "[audit] ERROR: benchmark calibration workflow must remain artifact-only" >&2 + check_status=1 + else + echo "[audit] ok: benchmark calibration workflow cannot commit, push, release, or open a pull request" + fi + if grep -Eq 'scripts/(benchmark_gate\.py (sample|compare)|paired_benchmark_gate\.py)' "$WORKFLOW_FILE"; then + echo "[audit] ERROR: required CI contains an unauthorized benchmark sampler, comparator, or paired gate" >&2 + check_status=1 + else + echo "[audit] ok: required CI uses only the authorized integrity and advisory interfaces" + fi + if grep -Eqi 'paired_benchmark_gate|benchmark-paired|paired[ _-]benchmark' "$WORKFLOW_FILE"; then + echo "[audit] ERROR: required CI contains a premature paired benchmark job or dependency" >&2 + check_status=1 + else + echo "[audit] ok: required CI contains no paired benchmark job or dependency" + fi require_pattern "$WORKFLOW_FILE" '^ push:$' 'CI push trigger' || check_status=1 require_pattern "$WORKFLOW_FILE" '^\s+- main$' 'CI push branch retains main' || check_status=1 require_pattern "$WORKFLOW_FILE" '^\s+- release/\*\*$' 'CI push branch includes release/**' || check_status=1 @@ -145,11 +682,11 @@ check_local_workflow() { upload_v5_count=$(grep -c 'actions/upload-artifact@v5' "$WORKFLOW_FILE" || true) upload_v6_count=$(grep -c 'actions/upload-artifact@v6' "$WORKFLOW_FILE" || true) upload_v7_count=$(grep -c 'actions/upload-artifact@v7' "$WORKFLOW_FILE" || true) - if [[ "$upload_v7_count" -ne 5 || "$upload_v5_count" -ne 0 || "$upload_v6_count" -ne 0 ]]; then - echo "[audit] ERROR: Phase 6 requires exactly five upload-artifact@v7 uses and zero v5/v6 uses" >&2 + if [[ "$upload_v7_count" -ne 6 || "$upload_v5_count" -ne 0 || "$upload_v6_count" -ne 0 ]]; then + echo "[audit] ERROR: required CI expects exactly six upload-artifact@v7 uses and zero v5/v6 uses" >&2 check_status=1 else - echo "[audit] ok: CI artifact uploads use actions/upload-artifact@v7 exactly five times" + echo "[audit] ok: CI artifact uploads use actions/upload-artifact@v7 exactly six times" fi quality_block="$(extract_job_block quality)" if [[ -z "$quality_block" ]]; then @@ -157,6 +694,8 @@ check_local_workflow() { check_status=1 else quality_checkout_block="$(extract_step_block_from_content "$quality_block" "Checkout")" + quality_plain_block="$(extract_step_block_from_content "$quality_block" "Test packages (plain codec)")" + quality_aes_gcm_block="$(extract_step_block_from_content "$quality_block" "Test packages (aes-gcm codec)")" validator_test_block="$(extract_step_block_from_content "$quality_block" "Test release-state validator")" validator_real_block="$(extract_step_block_from_content "$quality_block" "Validate repository release state")" require_content_pattern "$quality_checkout_block" '^ uses: actions/checkout@v6$' 'quality checkout uses actions/checkout@v6' || check_status=1 @@ -170,6 +709,10 @@ check_local_workflow() { require_content_pattern "$validator_real_block" 'refs/tags/v' 'release-state validator tag condition' || check_status=1 require_content_pattern "$validator_real_block" 'github\.event_name == .pull_request.' 'release-state validator pull-request condition' || check_status=1 require_content_pattern "$validator_real_block" 'github\.head_ref' 'release-state validator pull-request head condition' || check_status=1 + require_content_pattern "$quality_plain_block" '^ COLDKEEP_CODEC: plain$' 'SQLite quality plain codec environment' || check_status=1 + require_content_pattern "$quality_plain_block" '^ run: go test -race -count=1 \./cmd/\.\.\. \./internal/\.\.\.$' 'SQLite quality plain package command' || check_status=1 + require_content_pattern "$quality_aes_gcm_block" '^ COLDKEEP_CODEC: aes-gcm$' 'SQLite quality AES-GCM codec environment' || check_status=1 + require_content_pattern "$quality_aes_gcm_block" '^ run: go test -race -count=1 \./cmd/\.\.\. \./internal/\.\.\.$' 'SQLite quality AES-GCM package command' || check_status=1 if grep -Eq 'continue-on-error|\|\| true' <<<"$validator_test_block$validator_real_block"; then echo "[audit] ERROR: release-state validator steps must remain blocking" >&2 check_status=1 @@ -178,8 +721,104 @@ check_local_workflow() { fi fi require_pattern "$WORKFLOW_FILE" 'needs:\s*\[quality, correctness-matrix\]' 'smoke job depends on quality and correctness-matrix' || check_status=1 + correctness_matrix_block="$(extract_job_block correctness-matrix)" + if [[ -z "$correctness_matrix_block" ]]; then + echo "[audit] ERROR: missing correctness-matrix job block content" >&2 + check_status=1 + else + correctness_integration_block="$(extract_step_block_from_content "$correctness_matrix_block" "Run integration tests (correctness tier)")" + if [[ -z "$correctness_integration_block" ]]; then + echo "[audit] ERROR: missing integration correctness execution-proof step block" >&2 + check_status=1 + else + require_content_pattern "$correctness_integration_block" 'COLDKEEP_TEST_DB:\s*1' 'integration correctness execution proof enables DB gate' || check_status=1 + require_content_pattern "$correctness_integration_block" 'go test -race -count=1 -short -json \./tests/integration/\.\.\.' 'integration correctness execution proof uses JSON evidence' || check_status=1 + require_content_pattern "$correctness_integration_block" 'TestRoundTripStoreRestore' 'required PostgreSQL storage round-trip execution proof' || check_status=1 + require_content_pattern "$correctness_integration_block" 'TestRemoveWithSharedChunksRefCount' 'required PostgreSQL storage remove execution proof' || check_status=1 + require_content_pattern "$correctness_integration_block" 'TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState' 'required PostgreSQL recovery execution proof' || check_status=1 + require_content_pattern "$correctness_integration_block" 'github.com/franchoy/coldkeep/tests/integration' 'integration correctness execution proof binds the integration package' || check_status=1 + require_content_pattern "$correctness_integration_block" 'if codec == "plain"' 'integration correctness execution proof scopes recovery and remove markers to plain codec' || check_status=1 + require_content_pattern "$correctness_integration_block" 'json\.loads\(raw_line\)' 'integration correctness execution proof rejects malformed JSON' || check_status=1 + require_content_pattern "$correctness_integration_block" 'if not events:' 'integration correctness execution proof rejects empty JSON' || check_status=1 + require_content_pattern "$correctness_integration_block" 'event\.get\("Action"\) == "skip"' 'integration correctness execution proof rejects required skips' || check_status=1 + require_content_pattern "$correctness_integration_block" 'event\.get\("Action"\) == "pass"' 'integration correctness execution proof requires pass events' || check_status=1 + require_content_pattern "$correctness_integration_block" 'print\("required execution-proof failure:", file=sys\.stderr\)' 'integration correctness execution-proof parser' || check_status=1 + require_content_pattern "$correctness_integration_block" 'status=\$\{PIPESTATUS\[0\]\}' 'integration correctness execution proof preserves test status' || check_status=1 + require_content_pattern "$correctness_integration_block" 'status=\$\?' 'integration correctness execution proof propagates parser status' || check_status=1 + # shellcheck disable=SC2016 # The audit pattern must match the literal $status. + require_content_pattern "$correctness_integration_block" 'exit "\$status"' 'integration correctness execution proof remains blocking' || check_status=1 + if grep -Eq 'continue-on-error|go test .*\|\| true' <<<"$correctness_integration_block"; then + echo "[audit] ERROR: integration correctness execution-proof step must not suppress broad failures" >&2 + check_status=1 + else + echo "[audit] ok: integration correctness execution-proof step does not suppress broad failures" + fi + fi + postgres_internal_contracts_block="$(extract_step_block_from_content "$correctness_matrix_block" "Run required PostgreSQL internal package contracts")" + if [[ -z "$postgres_internal_contracts_block" ]]; then + echo "[audit] ERROR: missing required PostgreSQL internal package contracts step block" >&2 + check_status=1 + else + require_content_pattern "$postgres_internal_contracts_block" "^ - name: Run required PostgreSQL internal package contracts$" 'required PostgreSQL internal package contracts step' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" "if: \\$\\{\\{ matrix\.codec == 'plain' \\}\\}" 'PostgreSQL internal package contracts run only for plain codec' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'COLDKEEP_TEST_DB:\s*1' 'PostgreSQL internal package contracts enable DB gate' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'COLDKEEP_DB_AUTO_BOOTSTRAP:\s*true' 'PostgreSQL internal package contracts enable auto-bootstrap' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'DB_HOST:\s*127\.0\.0\.1' 'PostgreSQL internal package contracts set DB host' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'DB_PORT:\s*5432' 'PostgreSQL internal package contracts set DB port' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'DB_USER:\s*coldkeep' 'PostgreSQL internal package contracts set DB user' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'DB_PASSWORD:\s*coldkeep' 'PostgreSQL internal package contracts set DB password' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'DB_NAME:\s*coldkeep' 'PostgreSQL internal package contracts set DB name' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'DB_SSLMODE:\s*disable' 'PostgreSQL internal package contracts set DB SSL mode' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'go test -race -count=1 -json' 'PostgreSQL internal package contracts use race JSON test execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" './internal/testutil/backendtest' 'PostgreSQL internal package contracts include harness package' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" './internal/catalog' 'PostgreSQL internal package contracts include catalog package' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" './internal/db' 'PostgreSQL internal package contracts include DB package' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" './internal/engine' 'PostgreSQL internal package contracts include engine package' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" './internal/maintenance' 'PostgreSQL internal package contracts include maintenance package' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" './internal/container' 'PostgreSQL internal package contracts include container package' || check_status=1 + container_package_count="$(grep -Fc './internal/container' <<<"$postgres_internal_contracts_block")" + if [[ "$container_package_count" -ne 1 ]]; then + echo "[audit] ERROR: PostgreSQL internal package contracts must include ./internal/container exactly once (found $container_package_count)" >&2 + check_status=1 + else + echo "[audit] ok: PostgreSQL internal package contracts include container package exactly once" + fi + require_content_pattern "$postgres_internal_contracts_block" 'python3 - .*output_file' 'PostgreSQL internal package contracts parse JSON execution evidence' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'expected PostgreSQL pass missing' 'PostgreSQL internal package contracts require PostgreSQL test pass events' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestSCH001AndSCH002BootstrapVersionAndIdempotency/postgres' 'PostgreSQL internal package contracts prove Phase 5 bootstrap execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestSCH009PostgresVersionElevenAutoMigration/postgres' 'PostgreSQL internal package contracts prove Phase 5 migration execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadStatsAndInspectAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 stats and inspect execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadSnapshotViewsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 snapshot-view execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadVerifyAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 verification execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadContextAndErrorsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 context and error execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineSnapshotSelectorsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 8 selector execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineSnapshotSelectorErrorsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 8 selector error execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineMutationStoreRemoveAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 9 store/remove execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineMutationSnapshotLifecycleAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 9 snapshot lifecycle execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineMutationRestoreAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 9 restore execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineMutationErrorsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 9 error and rollback execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineGCDryRunAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 9 GC dry-run execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestBackendTransactionCommitRollbackAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 10 transaction execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestBackendForUpdateLockReleaseAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 10 FOR UPDATE execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestBackendNowaitAndSkipLockedAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 10 NOWAIT and SKIP LOCKED execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestBackendBlockedLockCancellationAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 10 blocked-lock cancellation execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestMutationRowsAffectedContractAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 17 mutation-cardinality execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestContainerRowLockIntegrationAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 10 container row-lock integration execution' || check_status=1 + if grep -Eq 'continue-on-error|\|\| true' <<<"$postgres_internal_contracts_block"; then + echo "[audit] ERROR: PostgreSQL internal package contracts step must remain blocking" >&2 + check_status=1 + else + echo "[audit] ok: PostgreSQL internal package contracts step is blocking" + fi + fi + fi require_pattern "$WORKFLOW_FILE" '^ cross-platform:$' 'cross-platform job exists' || check_status=1 require_pattern "$WORKFLOW_FILE" 'os:\s*\[ubuntu-latest, macos-latest, windows-latest\]' 'cross-platform job runs native ubuntu, macOS, and Windows matrix' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'name:\s*Run native coordination runtime tests' 'cross-platform native coordination runtime step' || check_status=1 + require_pattern "$WORKFLOW_FILE" "go test -v -count=1 -run '\\^\\(TestNativeLock\\|TestWindowsNativeLock\\|TestProductionCoordinator\\)' ./internal/coordination" 'cross-platform native coordination command covers native backends and production Coordinator' || check_status=1 + require_pattern "$NATIVE_UNIX_TEST_FILE" '^func TestNativeLockContentionAndReacquire' 'Unix native coordination source retains contention runtime test' || check_status=1 + require_pattern "$NATIVE_WINDOWS_TEST_FILE" '^func TestWindowsNativeLockContentionAndReacquire' 'Windows native coordination source retains contention runtime test' || check_status=1 + require_pattern "$COORDINATOR_NATIVE_TEST_FILE" '^func TestProductionCoordinatorsShareProcessRegistryAndProtectSuccessor' 'production Coordinator source retains registry and successor runtime test' || check_status=1 require_pattern "$WORKFLOW_FILE" 'name:\s*Run path safety cross-platform tests' 'cross-platform path safety step' || check_status=1 require_pattern "$WORKFLOW_FILE" "go test ./internal/pathsafe/\\.\\.\\. -run 'TrustedRoot\\|Symlink\\|Alias\\|WritePath' -count=1" 'cross-platform path safety command covers trusted-root and alias checks' || check_status=1 require_pattern "$WORKFLOW_FILE" 'name:\s*Run storage restore cross-platform tests' 'cross-platform storage restore step' || check_status=1 @@ -188,7 +827,7 @@ check_local_workflow() { require_pattern "$WORKFLOW_FILE" "go test ./internal/engine/\\.\\.\\. -run '\\^TestRestore' -count=1" 'cross-platform engine restore command scopes to restore tests' || check_status=1 require_pattern "$WORKFLOW_FILE" 'name:\s*Run snapshot restore cross-platform tests' 'cross-platform snapshot restore step' || check_status=1 require_pattern "$WORKFLOW_FILE" "go test ./internal/snapshot/\\.\\.\\. -run '\\^TestRestoreSnapshot' -count=1" 'cross-platform snapshot restore command scopes to snapshot restore tests' || check_status=1 - require_pattern "$WORKFLOW_FILE" 'needs:\s*\[quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-matrix, cross-platform\]' 'required gate depends on all upstream jobs including long-run, adversarial, legacy compatibility, benchmark matrix, and cross-platform' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'needs:\s*\[quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-integrity, benchmark-timing-advisory, cross-platform\]' 'required gate depends separately on benchmark integrity and timing advisory evaluation' || check_status=1 require_pattern "$WORKFLOW_FILE" 'if:\s*\$\{\{ always\(\) \}\}' 'required gate always evaluates upstream results' || check_status=1 require_pattern "$WORKFLOW_FILE" 'name:\s*Check smart quotes in Go files' 'smart-quote guard step' || check_status=1 require_pattern "$WORKFLOW_FILE" 'run:\s*bash scripts/check_smart_quotes\.sh' 'smart-quote guard command' || check_status=1 @@ -206,13 +845,14 @@ check_local_workflow() { require_pattern "$WORKFLOW_FILE" '^ integration-long-run:$' 'integration long-run job' || check_status=1 require_pattern "$WORKFLOW_FILE" '^ adversarial:$' 'adversarial job exists' || check_status=1 require_pattern "$WORKFLOW_FILE" 'name:\s*Run adversarial validation \(G1.*G17\)' 'adversarial workflow step names batch coverage through G17' || check_status=1 - require_pattern "$WORKFLOW_FILE" 'go test -race -count=1 ./tests/adversarial/\.\.\.' 'adversarial job targets adversarial suite' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'go test -race -count=1 -json ./tests/adversarial/\.\.\.' 'adversarial job targets adversarial suite with JSON evidence' || check_status=1 require_pattern "$WORKFLOW_FILE" "go test -race -count=1 ./tests/adversarial/... -run 'TestAdversarialG14\\|TestAdversarialG15\\|TestAdversarialG16\\|TestAdversarialG17'" 'explicit G14-G17 adversarial gate command' || check_status=1 adversarial_block="$(extract_job_block adversarial)" if [[ -z "$adversarial_block" ]]; then echo "[audit] ERROR: missing adversarial job block content" >&2 check_status=1 else + require_content_pattern "$adversarial_block" '^ runs-on: ubuntu-latest$' 'adversarial coordination proof runs on Linux' || check_status=1 require_content_pattern "$adversarial_block" '^ services:$' 'adversarial job declares services' || check_status=1 require_content_pattern "$adversarial_block" '^ postgres:$' 'adversarial job provisions postgres service' || check_status=1 require_content_pattern "$adversarial_block" 'image:\s*postgres:16' 'adversarial job pins postgres service image' || check_status=1 @@ -229,6 +869,35 @@ check_local_workflow() { require_content_pattern "$deterministic_g6_block" 'go test -v -race -count=1 ./tests/adversarial/\.\.\.' 'deterministic G6 PostgreSQL regression targets adversarial package explicitly' || check_status=1 require_content_pattern "$deterministic_g6_block" "-run '\\^TestAdversarialG6DeterministicStoreInterleavingPostgres\\$'" 'deterministic G6 PostgreSQL regression uses exact selector' || check_status=1 fi + adversarial_validation_block="$(extract_step_block_from_content "$adversarial_block" "Run adversarial validation (G1–G17)")" + if [[ -z "$adversarial_validation_block" ]]; then + echo "[audit] ERROR: missing adversarial coordination execution-proof step block" >&2 + check_status=1 + else + require_content_pattern "$adversarial_validation_block" 'COLDKEEP_TEST_DB:\s*1' 'adversarial coordination proof enables DB gate' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'COLDKEEP_LONG_RUN:\s*1' 'adversarial coordination proof enables long-run gate' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'go test -race -count=1 -json \./tests/adversarial/\.\.\.' 'adversarial coordination proof uses JSON execution evidence' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'TestAdversarialG6IndependentProcessRepositoryContention/plain' 'independent-process plain execution proof' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'TestAdversarialG6IndependentProcessRepositoryContention/aes-gcm' 'independent-process AES-GCM execution proof' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'TestAdversarialG6KilledLeaseHolderReleasesRepository' 'killed-holder execution proof' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'TestAdversarialG6LiveGCExcludesIndependentStoreProcess' 'live-GC execution proof' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'github.com/franchoy/coldkeep/tests/adversarial' 'adversarial coordination execution proof binds the adversarial package' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'json\.loads\(raw_line\)' 'adversarial coordination execution proof rejects malformed JSON' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'if not events:' 'adversarial coordination execution proof rejects empty JSON' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'event\.get\("Action"\) == "skip"' 'adversarial coordination execution proof rejects required skips' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'event\.get\("Action"\) == "pass"' 'adversarial coordination execution proof requires pass events' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'print\("required execution-proof failure:", file=sys\.stderr\)' 'adversarial coordination execution-proof parser' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'status=\$\{PIPESTATUS\[0\]\}' 'adversarial coordination proof preserves test status' || check_status=1 + require_content_pattern "$adversarial_validation_block" 'status=\$\?' 'adversarial coordination proof propagates parser status' || check_status=1 + # shellcheck disable=SC2016 # The audit pattern must match the literal $status. + require_content_pattern "$adversarial_validation_block" 'exit "\$status"' 'adversarial coordination proof remains blocking' || check_status=1 + if grep -Eq 'continue-on-error|\|\| true' <<<"$adversarial_validation_block"; then + echo "[audit] ERROR: adversarial coordination execution-proof step must not suppress broad failures" >&2 + check_status=1 + else + echo "[audit] ok: adversarial coordination execution-proof step does not suppress broad failures" + fi + fi fi require_pattern "$WORKFLOW_FILE" '^ smoke:$' 'smoke job' || check_status=1 require_pattern "$WORKFLOW_FILE" 'name:\s*Upload smoke artifacts on failure' 'smoke failure artifact upload step' || check_status=1 @@ -243,7 +912,8 @@ check_local_workflow() { require_pattern "$WORKFLOW_FILE" 'INTEGRATION_LONG_RUN_RESULT.*!= "success"' 'required gate rejects skipped integration long-run job' || check_status=1 require_pattern "$WORKFLOW_FILE" 'ADVERSARIAL_RESULT.*!= "success"' 'required gate rejects skipped adversarial job' || check_status=1 require_pattern "$WORKFLOW_FILE" 'SMOKE_RESULT.*!= "success"' 'required gate rejects skipped smoke job' || check_status=1 - require_pattern "$WORKFLOW_FILE" 'BENCHMARK_RESULT.*!= "success"' 'required gate rejects skipped benchmark job' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'BENCHMARK_INTEGRITY_RESULT.*!= "success"' 'required gate rejects skipped benchmark integrity job' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'BENCHMARK_TIMING_ADVISORY_RESULT.*!= "success"' 'required gate rejects skipped benchmark timing advisory job' || check_status=1 require_pattern "$WORKFLOW_FILE" 'CROSS_PLATFORM_RESULT.*!= "success"' 'required gate rejects skipped cross-platform job' || check_status=1 require_pattern "$CODEQL_WORKFLOW_FILE" 'name:\s*CodeQL' 'CodeQL workflow file' || check_status=1 require_pattern "$CODEQL_WORKFLOW_FILE" '^ push:$' 'CodeQL push trigger' || check_status=1 @@ -508,6 +1178,9 @@ status=0 if [[ "$REMOTE_ONLY" -eq 0 ]]; then check_local_workflow || status=1 + if [[ -n "$PAIRED_LAUNCHER_FILE" ]]; then + check_paired_launcher "$PAIRED_LAUNCHER_FILE" || status=1 + fi fi if [[ "$LOCAL_ONLY" -eq 0 ]]; then diff --git a/scripts/audit_ci_enforcement_test.go b/scripts/audit_ci_enforcement_test.go index d91c6dbe..7c16e78e 100644 --- a/scripts/audit_ci_enforcement_test.go +++ b/scripts/audit_ci_enforcement_test.go @@ -16,25 +16,715 @@ func TestAuditCIEnforcementLocalWorkflowRequiresCrossPlatformInNeeds(t *testing. codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) workflow = strings.Replace( workflow, - "needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-matrix, cross-platform]", - "needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-matrix]", + "needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-integrity, benchmark-timing-advisory, cross-platform]", + "needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-integrity, benchmark-timing-advisory]", 1, ) stderr := runAuditLocalOnly(t, workflow, codeqlWorkflow, true) - if !strings.Contains(stderr, "required gate depends on all upstream jobs") { + if !strings.Contains(stderr, "required gate depends separately on benchmark integrity") { t.Fatalf("expected missing cross-platform dependency error, got:\n%s", stderr) } } +func TestAuditCIEnforcementLocalWorkflowRequiresNativeCoordinationRuntime(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + + for name, test := range map[string]struct { + old string + replacement string + wantMessage string + }{ + "missing step": { + old: " - name: Run native coordination runtime tests\n" + + " run: go test -v -count=1 -run '^(TestNativeLock|TestWindowsNativeLock|TestProductionCoordinator)' ./internal/coordination\n\n", + wantMessage: "cross-platform native coordination runtime step", + }, + "altered command": { + old: "go test -v -count=1 -run '^(TestNativeLock|TestWindowsNativeLock|TestProductionCoordinator)' ./internal/coordination", + replacement: "go test -v -count=1 ./internal/coordination", + wantMessage: "cross-platform native coordination command covers native backends and production Coordinator", + }, + } { + t.Run(name, func(t *testing.T) { + mutated := strings.Replace(workflow, test.old, test.replacement, 1) + if mutated == workflow { + t.Fatalf("workflow fixture did not contain %q", test.old) + } + stderr := runAuditLocalOnly(t, mutated, codeqlWorkflow, true) + if !strings.Contains(stderr, test.wantMessage) { + t.Fatalf("expected %q, got:\n%s", test.wantMessage, stderr) + } + }) + } +} + +func TestAuditCIEnforcementRequiresPinnedBenchmarkCalibrationToolchain(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + baselineWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "benchmark-baseline.yml")) + baselineWorkflow = strings.Replace(baselineWorkflow, "go-version: '1.25.12'", "go-version: '1.25.x'", 1) + + stderr := runAuditLocalOnlyWithBaseline(t, workflow, codeqlWorkflow, baselineWorkflow, true) + if !strings.Contains(stderr, "benchmark calibration pins the Go patch") { + t.Fatalf("expected benchmark calibration toolchain error, got:\n%s", stderr) + } +} + +func TestAuditCIEnforcementRejectsUnsafeBenchmarkCalibrationWorkflow(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + baselineWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "benchmark-baseline.yml")) + tests := []struct { + name string + mutate func(string) string + wantMessage string + }{ + { + name: "main-only authorization removed", + mutate: func(value string) string { + return strings.Replace( + value, + `if [[ "${TRUSTED_REF}" != "refs/heads/main" ]]; then`, + `if [[ "${TRUSTED_REF}" != "refs/heads/release/v1.13.11" ]]; then`, + 1, + ) + }, + wantMessage: "benchmark authorization is fail-closed on refs/heads/main", + }, + { + name: "source SHA equality removed", + mutate: func(value string) string { + return strings.Replace( + value, + `if [[ "${SOURCE_SHA}" != "${TRUSTED_SHA}" ]]; then`, + `if [[ -z "${SOURCE_SHA}" ]]; then`, + 1, + ) + }, + wantMessage: "benchmark source_sha must equal trusted github.sha", + }, + { + name: "checkout source made operator-controlled", + mutate: func(value string) string { + return strings.Replace(value, "ref: ${{ github.sha }}", "ref: ${{ inputs.source_sha }}", 1) + }, + wantMessage: "benchmark checkout cannot use inputs.source_sha", + }, + { + name: "persisted credentials defaulted", + mutate: func(value string) string { + return strings.Replace(value, " persist-credentials: false\n", "", 1) + }, + wantMessage: "benchmark checkouts must disable persisted credentials", + }, + { + name: "setup-go cache re-enabled", + mutate: func(value string) string { + return strings.Replace(value, " cache: false", " cache: true", 1) + }, + wantMessage: "benchmark setup-go caching must be disabled", + }, + { + name: "sample harness redirected", + mutate: func(value string) string { + return strings.Replace( + value, + "python3 scripts/benchmark_gate.py sample", + "python3 governed-source/scripts/benchmark_gate.py sample", + 1, + ) + }, + wantMessage: "benchmark sample harness runs from trusted checkout", + }, + { + name: "calibration harness redirected", + mutate: func(value string) string { + return strings.Replace( + value, + "python3 scripts/benchmark_gate.py calibrate", + "python3 governed-source/scripts/benchmark_gate.py calibrate", + 1, + ) + }, + wantMessage: "benchmark calibration harness runs from trusted checkout", + }, + { + name: "runner temp artifact isolation removed", + mutate: func(value string) string { + return strings.Replace( + value, + "path: ${{ runner.temp }}/benchmark-calibration-input", + "path: downloaded", + 1, + ) + }, + wantMessage: "benchmark calibration artifacts use runner.temp", + }, + { + name: "artifact source provenance check removed", + mutate: func(value string) string { + return strings.Replace(value, " if actual != expected:\n", "", 1) + }, + wantMessage: "benchmark calibration requires artifact provenance to match github.sha", + }, + { + name: "authorization failure suppressed", + mutate: func(value string) string { + return strings.Replace(value, " set -euo pipefail", " set -euo pipefail\n set +e", 1) + }, + wantMessage: "benchmark source validation must not use broad failure suppression", + }, + { + name: "automatic schedule", + mutate: func(value string) string { + return strings.Replace(value, " workflow_dispatch:", " schedule:\n workflow_dispatch:", 1) + }, + wantMessage: "benchmark calibration workflow must remain manual-only", + }, + { + name: "write permission", + mutate: func(value string) string { + return strings.Replace(value, " contents: read", " contents: write", 1) + }, + wantMessage: "benchmark calibration workflow must not receive write permission", + }, + { + name: "adaptive sample count", + mutate: func(value string) string { + return strings.Replace(value, " sample_count=10", " sample_count=11", 1) + }, + wantMessage: "benchmark calibration fixes ten measured samples", + }, + { + name: "fixture drift", + mutate: func(value string) string { + return strings.Replace(value, " --dataset ci-stable-v1", " --dataset small", 1) + }, + wantMessage: "benchmark calibration fixes the fixture identity", + }, + { + name: "push step", + mutate: func(value string) string { + return value + "\n# git push origin HEAD\n" + }, + wantMessage: "benchmark calibration workflow must remain artifact-only", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutated := tt.mutate(baselineWorkflow) + if mutated == baselineWorkflow { + t.Fatal("benchmark baseline workflow mutation target not found") + } + stderr := runAuditLocalOnlyWithBaseline( + t, + workflow, + codeqlWorkflow, + mutated, + true, + ) + if !strings.Contains(stderr, tt.wantMessage) { + t.Fatalf("expected %q, got:\n%s", tt.wantMessage, stderr) + } + }) + } +} + +func TestAuditCIEnforcementRejectsBenchmarkGovernanceMutations(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + tests := []struct { + name string + old string + replacement string + message string + }{ + { + name: "missing integrity profile", old: " - profile: none-w4\n", + replacement: "", message: "integrity matrix must contain profile none-w4 exactly once", + }, + { + name: "fixture drift", old: "dataset: ci-paired-w4-v2", + replacement: "dataset: ci-paired-w4-v1", message: "bounded workers=4 fixture", + }, + { + name: "timeout drift", old: "--command-timeout-seconds 600", + replacement: "--command-timeout-seconds 601", message: "600-second command timeout", + }, + { + name: "integrity downgrade", old: "python3 scripts/benchmark_gate.py integrity", + replacement: "python3 scripts/validate_regression_thresholds.py check", message: "hard candidate-only interface", + }, + { + name: "advisory made legacy", old: "--policy hosted-advisory", + replacement: "--policy legacy", message: "informational authority", + }, + { + name: "exit verifier removed", old: "verify-advisory-exit", + replacement: "verify-removed-exit", message: "exact classification and exit code", + }, + { + name: "machine report not required", old: " [[ -s \"${report}\" ]]\n", + replacement: "", message: "requires a machine-readable report", + }, + { + name: "advisory exit allowlist widened", old: " 0|10|11|12)\n", + replacement: " 0|2|10|11|12)\n", message: "narrowly accepts valid informational exit codes", + }, + { + name: "evaluator failure made successful", old: " 2)\n exit 2\n", + replacement: " 2)\n exit 0\n", message: "must return failure for evaluator exit code 2", + }, + { + name: "timing checksum verification removed", old: " sha256sum --check checksums.sha256\n", + replacement: "", message: "timing artifact verifies checksums", + }, + { + name: "broad suppression", old: " report=\"${evidence_dir}/timing-advisory.json\"\n set +e\n", + replacement: " report=\"${evidence_dir}/timing-advisory.json\"\n continue-on-error: true\n set +e\n", message: "broad failure suppression", + }, + { + name: "integrity missing artifact allowed", old: " if-no-files-found: error\n\n benchmark-timing-advisory:", + replacement: " if-no-files-found: ignore\n\n benchmark-timing-advisory:", message: "integrity artifact rejects missing evidence", + }, + { + name: "advisory upload not always", old: " - name: Upload benchmark timing advisory evidence\n if: ${{ always() }}", + replacement: " - name: Upload benchmark timing advisory evidence\n if: ${{ success() }}", message: "timing artifact upload always runs", + }, + { + name: "required dependency removed", old: "benchmark-integrity, benchmark-timing-advisory, cross-platform", + replacement: "benchmark-timing-advisory, cross-platform", message: "depends separately on benchmark integrity", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutated := strings.Replace(workflow, tt.old, tt.replacement, 1) + if mutated == workflow { + t.Fatalf("mutation target not found: %q", tt.old) + } + output := runAuditLocalOnly(t, mutated, codeqlWorkflow, true) + if !strings.Contains(output, tt.message) { + t.Fatalf("expected %q, got:\n%s", tt.message, output) + } + }) + } +} + +func TestAuditCIEnforcementRequiresSeparatedTimingAndIntegrityContracts(t *testing.T) { + validator := readRepoFile(t, filepath.Join("scripts", "validate_regression_thresholds.py")) + tests := []struct { + name string + old string + replacement string + message string + }{ + { + name: "diagnostic state made required", + old: `TIMING_ROW_OPTIONAL_FIELDS = {"diagnostic_final_state"}`, + replacement: `TIMING_ROW_OPTIONAL_FIELDS = {}`, + message: "historical timing treats diagnostic final state as optional", + }, + { + name: "optional state no longer validated", + old: `not legacy and "diagnostic_final_state" in row`, + replacement: `not legacy and False`, + message: "optional timing diagnostic final state is validated when present", + }, + { + name: "hard state imported into timing", + old: `TIMING_ROW_OPTIONAL_FIELDS = {"diagnostic_final_state"}`, + replacement: "TIMING_ROW_OPTIONAL_FIELDS = {\"diagnostic_final_state\"}\nbenchmark_contract.hard_final_state({})", + message: "historical timing advisory must not require hard diagnostic final state", + }, + { + name: "evaluator exit remapped", + old: `"BENCHMARK_TIMING_EVALUATION_FAILURE": 2`, + replacement: `"BENCHMARK_TIMING_EVALUATION_FAILURE": 12`, + message: "timing evaluator failure maps exactly to exit code 2", + }, + { + name: "omitempty counter removed", + old: `"container_append_count", "fsync_count", "container_open_count",`, + replacement: `"fsync_count", "container_open_count",`, + message: "timing validator models Go omitempty field container_append_count", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutated := strings.Replace(validator, tt.old, tt.replacement, 1) + if mutated == validator { + t.Fatalf("mutation target not found: %q", tt.old) + } + output := runAuditFixtureWithTimingValidator( + t, + readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")), + readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")), + readRepoFile(t, filepath.Join(".github", "workflows", "benchmark-baseline.yml")), + mutated, + true, + false, + false, + ) + if !strings.Contains(output, tt.message) { + t.Fatalf("expected %q, got:\n%s", tt.message, output) + } + }) + } +} + +func TestAuditCIEnforcementRejectsPrematureRequiredBenchmarkGateSwitch(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + workflow = strings.Replace( + workflow, + "python3 scripts/validate_regression_thresholds.py check", + "python3 scripts/benchmark_gate.py compare", + 1, + ) + stderr := runAuditLocalOnly( + t, + workflow, + readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")), + true, + ) + if !strings.Contains(stderr, "unauthorized benchmark sampler, comparator, or paired gate") { + t.Fatalf("expected premature gate-switch error, got:\n%s", stderr) + } +} + +func TestAuditCIEnforcementRejectsPrematurePairedBenchmarkGateSwitch(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + workflow = strings.Replace( + workflow, + "python3 scripts/validate_regression_thresholds.py check", + "python3 scripts/paired_benchmark_gate.py sample", + 1, + ) + stderr := runAuditLocalOnly( + t, + workflow, + readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")), + true, + ) + if !strings.Contains(stderr, "unauthorized benchmark sampler, comparator, or paired gate") { + t.Fatalf("expected premature paired gate-switch error, got:\n%s", stderr) + } +} + +func TestAuditCIEnforcementRejectsPrematurePairedBenchmarkDependency(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + workflow = strings.Replace( + workflow, + "needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-integrity, benchmark-timing-advisory, cross-platform]", + "needs: [quality, correctness-matrix, integration-stress, integration-long-run, adversarial, smoke, legacy-compatibility, benchmark-integrity, benchmark-timing-advisory, benchmark-paired-decision, cross-platform]", + 1, + ) + stderr := runAuditLocalOnly( + t, + workflow, + readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")), + true, + ) + if !strings.Contains(stderr, "required CI contains a premature paired benchmark job or dependency") { + t.Fatalf("expected premature paired dependency error, got:\n%s", stderr) + } +} + +func TestAuditCIEnforcementRejectsPrematurePairedGovernanceFiles(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + baselineWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "benchmark-baseline.yml")) + for _, test := range []struct { + name string + reference bool + threshold bool + message string + }{ + {name: "reference", reference: true, message: "paired reference manifest exists before governance authorization"}, + {name: "threshold", threshold: true, message: "paired threshold policy exists before threshold authorization"}, + } { + t.Run(test.name, func(t *testing.T) { + stderr := runAuditFixture( + t, + workflow, + codeqlWorkflow, + baselineWorkflow, + true, + test.reference, + test.threshold, + ) + if !strings.Contains(stderr, test.message) { + t.Fatalf("expected %q, got:\n%s", test.message, stderr) + } + }) + } +} + +func TestAuditCIEnforcementPairedLauncherConfidentialityAndLifecycle(t *testing.T) { + compliant := `name: Temporary Paired Diagnostic +jobs: + sample: + timeout-minutes: 45 + strategy: + matrix: + include: + - profile: none-w1 + dataset: ci-paired-w1-v2 + - profile: none-w4 + dataset: ci-paired-w4-v2 + - profile: zstd-w1 + dataset: ci-paired-w1-v2 + - profile: zstd-w4 + dataset: ci-paired-w4-v2 + steps: + - name: Mask runner roots + run: | + set +x + echo "::add-mask::$GITHUB_WORKSPACE" + echo "::add-mask::$RUNNER_TEMP" + echo "::add-mask::$HOME" + echo '/github/workspace /github/runner_temp' + - name: Sample + run: | + set +x + token="$(openssl rand -hex 12)" + echo "::add-mask::${token}" + sensitive_root="$(mktemp -d "${RUNNER_TEMP}/paired.XXXXXXXX")" + echo "::add-mask::${sensitive_root}" + profile_parent="${GITHUB_WORKSPACE}/paired-evidence/${{ matrix.profile }}" + profile_output="${profile_parent}/artifact" + mkdir -p "${profile_parent}" + test ! -e "${profile_output}" + python3 scripts/paired_benchmark_gate.py sample \ + --dataset "${{ matrix.dataset }}" \ + --pairs 10 \ + --command-timeout-seconds 600 \ + --output-dir "${profile_output}" + - name: Upload profile + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + path: paired-evidence/${{ matrix.profile }}/artifact + decision: + timeout-minutes: 10 + steps: + - name: Decide + run: | + set +x + decision_parent="${GITHUB_WORKSPACE}/paired-decision" + decision_output="${decision_parent}/decision" + mkdir -p "${decision_parent}" + test ! -e "${decision_output}" + python3 scripts/paired_benchmark_gate.py decision \ + --mode diagnostic \ + --output-dir "${decision_output}" + - name: Upload decision + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + path: paired-decision/decision +` + tests := []struct { + name string + mutate func(string) string + wantFailure bool + message string + }{ + {name: "parent-only creation", mutate: func(value string) string { return value }}, + {name: "distinct child per profile", mutate: func(value string) string { return value }}, + {name: "nonexistent decision child", mutate: func(value string) string { return value }}, + {name: "exact harness-owned upload", mutate: func(value string) string { return value }}, + {name: "platform aliases", mutate: func(value string) string { return value }}, + {name: "generated values masked", mutate: func(value string) string { return value }}, + {name: "no governance authority", mutate: func(value string) string { return value }}, + { + name: "pre-created sample output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"", " touch \"${profile_output}\"\n test ! -e \"${profile_output}\"", 1) + }, + wantFailure: true, + message: "sample output must not be created before harness invocation", + }, + { + name: "pre-created decision output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${decision_output}\"", " touch \"${decision_output}\"\n test ! -e \"${decision_output}\"", 1) + }, + wantFailure: true, + message: "decision output must not be created before harness invocation", + }, + { + name: "mkdir sample output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"", " mkdir -p \"${profile_output}\"\n test ! -e \"${profile_output}\"", 1) + }, + wantFailure: true, + message: "sample output must not be created before harness invocation", + }, + { + name: "mkdir decision output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${decision_output}\"", " mkdir -p \"${decision_output}\"\n test ! -e \"${decision_output}\"", 1) + }, + wantFailure: true, + message: "decision output must not be created before harness invocation", + }, + { + name: "install output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"", " install -d \"${profile_output}\"\n test ! -e \"${profile_output}\"", 1) + }, + wantFailure: true, + message: "sample output must not be created before harness invocation", + }, + { + name: "checkout into output", + mutate: func(value string) string { + needle := " - name: Sample\n" + checkout := " - name: Unsafe checkout\n uses: actions/checkout@v6\n with:\n path: paired-evidence/${{ matrix.profile }}/artifact\n" + return strings.Replace(value, needle, checkout+needle, 1) + }, + wantFailure: true, + message: "sample output must not be an actions/checkout destination", + }, + { + name: "extract into output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"", " tar -xf evidence.tar -C \"${profile_output}\"\n test ! -e \"${profile_output}\"", 1) + }, + wantFailure: true, + message: "sample output must not be populated, checked out, extracted, or recreated", + }, + { + name: "missing nonexistence assertion", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"\n", "", 1) + }, + wantFailure: true, + message: "sample output requires an exact nonexistence assertion", + }, + { + name: "upload mismatch", + mutate: func(value string) string { + return strings.Replace(value, " path: paired-evidence/${{ matrix.profile }}/artifact", " path: paired-evidence/${{ matrix.profile }}/different", 1) + }, + wantFailure: true, + message: "sample upload path must equal the harness-owned output path", + }, + { + name: "shared matrix output", + mutate: func(value string) string { + return strings.Replace(value, "profile_parent=\"${GITHUB_WORKSPACE}/paired-evidence/${{ matrix.profile }}\"", "profile_parent=\"${GITHUB_WORKSPACE}/paired-evidence/shared\"", 1) + }, + wantFailure: true, + message: "sample output must be distinct for every matrix profile", + }, + { + name: "workspace root output", + mutate: func(value string) string { + return strings.Replace(value, "profile_output=\"${profile_parent}/artifact\"", "profile_output=\"${GITHUB_WORKSPACE}\"", 1) + }, + wantFailure: true, + message: "sample output must be a nonexistent child below a contained parent", + }, + { + name: "traversal output", + mutate: func(value string) string { + return strings.Replace(value, "profile_output=\"${profile_parent}/artifact\"", "profile_output=\"${profile_parent}/../artifact\"", 1) + }, + wantFailure: true, + message: "sample output must not use traversal", + }, + { + name: "symlink output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"", " ln -s elsewhere \"${profile_output}\"\n test ! -e \"${profile_output}\"", 1) + }, + wantFailure: true, + message: "sample output must not be populated, checked out, extracted, or recreated", + }, + { + name: "delete and recreate output", + mutate: func(value string) string { + return strings.Replace(value, " test ! -e \"${profile_output}\"", " rm -rf \"${profile_output}\"\n mkdir -p \"${profile_output}\"\n test ! -e \"${profile_output}\"", 1) + }, + wantFailure: true, + message: "sample output must not be created before harness invocation", + }, + { + name: "yaml env exposure", + mutate: func(value string) string { + return value + " env:\n DB_PASSWORD: exposed\n" + }, + wantFailure: true, + message: "prohibited values through YAML env", + }, + { + name: "pre-mask service", + mutate: func(value string) string { + return strings.Replace(value, " steps:\n", " services:\n postgres:\n image: postgres:16\n steps:\n", 1) + }, + wantFailure: true, + message: "must provision its isolated container after masking", + }, + { + name: "outer timeout gap", + mutate: func(value string) string { + return strings.Replace(value, "timeout-minutes: 45", "timeout-minutes: 40", 1) + }, + wantFailure: true, + message: "outer timeout is 45 minutes", + }, + { + name: "runtime persistence", + mutate: func(value string) string { return value + "# GITHUB_ENV\n" }, + wantFailure: true, + message: "persists or traces sensitive runtime values", + }, + { + name: "generated path printed before masking", + mutate: func(value string) string { + return strings.Replace(value, " echo \"::add-mask::${sensitive_root}\"", " echo \"${sensitive_root}\"", 1) + }, + wantFailure: true, + message: "generated dynamic paths and identifiers must be masked before printing", + }, + { + name: "production authority", + mutate: func(value string) string { + return strings.Replace(value, "--mode diagnostic", "--mode production", 1) + }, + wantFailure: true, + message: "paired launcher must remain diagnostic-only", + }, + { + name: "threshold authority", + mutate: func(value string) string { + return value + "\n# threshold-policy-v1.13.json\n" + }, + wantFailure: true, + message: "must not create manifest or threshold authority", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output := runPairedLauncherAudit(t, tt.mutate(compliant), tt.wantFailure) + if tt.message != "" && !strings.Contains(output, tt.message) { + t.Fatalf("expected %q, got:\n%s", tt.message, output) + } + }) + } +} + func TestAuditCIEnforcementLocalWorkflowRequiresCrossPlatformSuccessAssertion(t *testing.T) { workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) workflow = strings.Replace( workflow, - ` [ "${BENCHMARK_RESULT}" != "success" ] || \ + ` [ "${BENCHMARK_TIMING_ADVISORY_RESULT}" != "success" ] || \ [ "${CROSS_PLATFORM_RESULT}" != "success" ]; then`, - ` [ "${BENCHMARK_RESULT}" != "success" ]; then`, + ` [ "${BENCHMARK_TIMING_ADVISORY_RESULT}" != "success" ]; then`, 1, ) @@ -97,6 +787,342 @@ func TestAuditCIEnforcementLocalWorkflowPassesCurrentConfiguration(t *testing.T) } } +func TestAuditCIEnforcementRejectsPhase18RequiredProofMutations(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + tests := []struct { + name string + anchor string + old string + replacement string + wantMessage string + sourcePath string + sourceEnv string + }{ + { + name: "SQLite plain package command", + anchor: " - name: Test packages (plain codec)\n", + old: "go test -race -count=1 ./cmd/... ./internal/...", + replacement: "go test -race -count=1 ./internal/...", + wantMessage: "SQLite quality plain package command", + }, + { + name: "SQLite AES-GCM package command", + anchor: " - name: Test packages (aes-gcm codec)\n", + old: "go test -race -count=1 ./cmd/... ./internal/...", + replacement: "go test -race -count=1 ./cmd/...", + wantMessage: "SQLite quality AES-GCM package command", + }, + { + name: "Phase 17 PostgreSQL mutation marker", + anchor: " - name: Run required PostgreSQL internal package contracts\n", + old: "TestMutationRowsAffectedContractAcrossBackends/postgres", + replacement: "TestMutationRowsAffectedContractAcrossBackends/sqlite", + wantMessage: "PostgreSQL internal package contracts prove Phase 17 mutation-cardinality execution", + }, + { + name: "Unix native contention source", + old: "func TestNativeLockContentionAndReacquire", + replacement: "func removedNativeLockContentionAndReacquire", + wantMessage: "Unix native coordination source retains contention runtime test", + sourcePath: filepath.Join("internal", "coordination", "native_lock_unix_test.go"), + sourceEnv: "COLDKEEP_NATIVE_UNIX_TEST_FILE", + }, + { + name: "Windows native contention source", + old: "func TestWindowsNativeLockContentionAndReacquire", + replacement: "func removedWindowsNativeLockContentionAndReacquire", + wantMessage: "Windows native coordination source retains contention runtime test", + sourcePath: filepath.Join("internal", "coordination", "native_lock_windows_test.go"), + sourceEnv: "COLDKEEP_NATIVE_WINDOWS_TEST_FILE", + }, + { + name: "production Coordinator source", + old: "func TestProductionCoordinatorsShareProcessRegistryAndProtectSuccessor", + replacement: "func removedProductionCoordinatorsShareProcessRegistryAndProtectSuccessor", + wantMessage: "production Coordinator source retains registry and successor runtime test", + sourcePath: filepath.Join("internal", "coordination", "coordinator_native_test.go"), + sourceEnv: "COLDKEEP_COORDINATOR_NATIVE_TEST_FILE", + }, + { + name: "correctness DB gate", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "COLDKEEP_TEST_DB: 1", + replacement: "COLDKEEP_TEST_DB: 0", + wantMessage: "integration correctness execution proof enables DB gate", + }, + { + name: "correctness JSON command", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "go test -race -count=1 -short -json ./tests/integration/...", + replacement: "go test -race -count=1 -short ./tests/integration/...", + wantMessage: "integration correctness execution proof uses JSON evidence", + }, + { + name: "storage round-trip marker", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "TestRoundTripStoreRestore", + replacement: "RemovedRoundTripMarker", + wantMessage: "required PostgreSQL storage round-trip execution proof", + }, + { + name: "storage remove marker", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "TestRemoveWithSharedChunksRefCount", + replacement: "RemovedSharedChunkMarker", + wantMessage: "required PostgreSQL storage remove execution proof", + }, + { + name: "startup recovery marker", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "TestStartupRecoveryResyncsPreexistingQuarantinedOrphanConflictState", + replacement: "TestStartupRecoveryMarkerRemoved", + wantMessage: "required PostgreSQL recovery execution proof", + }, + { + name: "correctness plain codec scope", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "if codec == \"plain\":", + replacement: "if codec == \"unused\":", + wantMessage: "integration correctness execution proof scopes recovery and remove markers to plain codec", + }, + { + name: "correctness package binding", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "github.com/franchoy/coldkeep/tests/integration", + replacement: "github.com/franchoy/coldkeep/tests/adversarial", + wantMessage: "integration correctness execution proof binds the integration package", + }, + { + name: "correctness malformed JSON rejection", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "json.loads(raw_line)", + replacement: "{}", + wantMessage: "integration correctness execution proof rejects malformed JSON", + }, + { + name: "correctness empty JSON rejection", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "if not events:", + replacement: "if False:", + wantMessage: "integration correctness execution proof rejects empty JSON", + }, + { + name: "correctness skip rejection", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "event.get(\"Action\") == \"skip\"", + replacement: "False", + wantMessage: "integration correctness execution proof rejects required skips", + }, + { + name: "correctness pass requirement", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "event.get(\"Action\") == \"pass\"", + replacement: "event.get(\"Action\") == \"output\"", + wantMessage: "integration correctness execution proof requires pass events", + }, + { + name: "correctness parser diagnostic", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "print(\"required execution-proof failure:\", file=sys.stderr)", + replacement: "print(\"execution proof failed\", file=sys.stderr)", + wantMessage: "integration correctness execution-proof parser", + }, + { + name: "correctness test status", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "status=${PIPESTATUS[0]}", + replacement: "status=0", + wantMessage: "integration correctness execution proof preserves test status", + }, + { + name: "correctness parser status", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "status=$?", + replacement: "status=0", + wantMessage: "integration correctness execution proof propagates parser status", + }, + { + name: "correctness blocking exit", + anchor: " - name: Run integration tests (correctness tier)\n", + old: "exit \"$status\"", + replacement: "exit 0", + wantMessage: "integration correctness execution proof remains blocking", + }, + { + name: "correctness broad failure suppression", + anchor: " - name: Run integration tests (correctness tier)\n", + old: " env:\n", + replacement: " continue-on-error: true\n env:\n", + wantMessage: "integration correctness execution-proof step must not suppress broad failures", + }, + { + name: "adversarial Linux runner", + anchor: " adversarial:\n", + old: "runs-on: ubuntu-latest", + replacement: "runs-on: macos-latest", + wantMessage: "adversarial coordination proof runs on Linux", + }, + { + name: "adversarial PostgreSQL service", + anchor: " adversarial:\n", + old: "image: postgres:16", + replacement: "image: postgres:15", + wantMessage: "adversarial job pins postgres service image", + }, + { + name: "adversarial DB gate", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "COLDKEEP_TEST_DB: 1", + replacement: "COLDKEEP_TEST_DB: 0", + wantMessage: "adversarial coordination proof enables DB gate", + }, + { + name: "adversarial long-run gate", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "COLDKEEP_LONG_RUN: 1", + replacement: "COLDKEEP_LONG_RUN: 0", + wantMessage: "adversarial coordination proof enables long-run gate", + }, + { + name: "adversarial JSON command", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "go test -race -count=1 -json ./tests/adversarial/...", + replacement: "go test -race -count=1 ./tests/adversarial/...", + wantMessage: "adversarial coordination proof uses JSON execution evidence", + }, + { + name: "independent-process plain marker", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "TestAdversarialG6IndependentProcessRepositoryContention/plain", + replacement: "TestAdversarialG6IndependentProcessRepositoryContention/removed", + wantMessage: "independent-process plain execution proof", + }, + { + name: "independent-process AES-GCM marker", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "TestAdversarialG6IndependentProcessRepositoryContention/aes-gcm", + replacement: "TestAdversarialG6IndependentProcessRepositoryContention/removed", + wantMessage: "independent-process AES-GCM execution proof", + }, + { + name: "killed-holder marker", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "TestAdversarialG6KilledLeaseHolderReleasesRepository", + replacement: "TestAdversarialG6KilledHolderMarkerRemoved", + wantMessage: "killed-holder execution proof", + }, + { + name: "live-GC marker", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "TestAdversarialG6LiveGCExcludesIndependentStoreProcess", + replacement: "TestAdversarialG6LiveGCMarkerRemoved", + wantMessage: "live-GC execution proof", + }, + { + name: "adversarial package binding", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "github.com/franchoy/coldkeep/tests/adversarial", + replacement: "github.com/franchoy/coldkeep/tests/integration", + wantMessage: "adversarial coordination execution proof binds the adversarial package", + }, + { + name: "adversarial malformed JSON rejection", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "json.loads(raw_line)", + replacement: "{}", + wantMessage: "adversarial coordination execution proof rejects malformed JSON", + }, + { + name: "adversarial empty JSON rejection", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "if not events:", + replacement: "if False:", + wantMessage: "adversarial coordination execution proof rejects empty JSON", + }, + { + name: "adversarial skip rejection", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "event.get(\"Action\") == \"skip\"", + replacement: "False", + wantMessage: "adversarial coordination execution proof rejects required skips", + }, + { + name: "adversarial pass requirement", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "event.get(\"Action\") == \"pass\"", + replacement: "event.get(\"Action\") == \"output\"", + wantMessage: "adversarial coordination execution proof requires pass events", + }, + { + name: "adversarial parser diagnostic", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "print(\"required execution-proof failure:\", file=sys.stderr)", + replacement: "print(\"execution proof failed\", file=sys.stderr)", + wantMessage: "adversarial coordination execution-proof parser", + }, + { + name: "adversarial test status", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "status=${PIPESTATUS[0]}", + replacement: "status=0", + wantMessage: "adversarial coordination proof preserves test status", + }, + { + name: "adversarial parser status", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "status=$?", + replacement: "status=0", + wantMessage: "adversarial coordination proof propagates parser status", + }, + { + name: "adversarial blocking exit", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: "exit \"$status\"", + replacement: "exit 0", + wantMessage: "adversarial coordination proof remains blocking", + }, + { + name: "adversarial broad failure suppression", + anchor: " - name: Run adversarial validation (G1–G17)\n", + old: " id: adversarial_g1_g17\n", + replacement: " id: adversarial_g1_g17\n continue-on-error: true\n", + wantMessage: "adversarial coordination execution-proof step must not suppress broad failures", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.sourcePath != "" { + source := readRepoFile(t, test.sourcePath) + mutated := strings.Replace(source, test.old, test.replacement, 1) + if mutated == source { + t.Fatalf("source fixture %s did not contain %q", test.sourcePath, test.old) + } + stderr := runAuditLocalOnlyWithSourceFixture(t, workflow, codeqlWorkflow, test.sourceEnv, mutated) + if !strings.Contains(stderr, test.wantMessage) { + t.Fatalf("expected %q, got:\n%s", test.wantMessage, stderr) + } + return + } + anchorIndex := strings.Index(workflow, test.anchor) + if anchorIndex < 0 { + t.Fatalf("workflow fixture did not contain anchor %q", test.anchor) + } + targetOffset := strings.Index(workflow[anchorIndex:], test.old) + if targetOffset < 0 { + t.Fatalf("workflow fixture did not contain %q after anchor %q", test.old, test.anchor) + } + targetIndex := anchorIndex + targetOffset + mutated := workflow[:targetIndex] + test.replacement + workflow[targetIndex+len(test.old):] + stderr := runAuditLocalOnly(t, mutated, codeqlWorkflow, true) + if !strings.Contains(stderr, test.wantMessage) { + t.Fatalf("expected %q, got:\n%s", test.wantMessage, stderr) + } + }) + } +} + func TestAuditCIEnforcementLocalWorkflowRequiresDeterministicG6PostgresCommand(t *testing.T) { workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) @@ -129,13 +1155,121 @@ func TestAuditCIEnforcementLocalWorkflowRequiresDeterministicG6DBGate(t *testing } } +func runAuditLocalOnlyWithSourceFixture( + t *testing.T, + workflow string, + codeqlWorkflow string, + sourceEnv string, + source string, +) string { + t.Helper() + sourcePath := filepath.Join(t.TempDir(), "coordination_test.go") + if err := os.WriteFile(sourcePath, []byte(source), 0o600); err != nil { + t.Fatalf("write coordination source fixture: %v", err) + } + t.Setenv(sourceEnv, sourcePath) + return runAuditLocalOnly(t, workflow, codeqlWorkflow, true) +} + func runAuditLocalOnly(t *testing.T, workflow string, codeqlWorkflow string, wantFailure bool) string { t.Helper() + return runAuditLocalOnlyWithBaseline( + t, + workflow, + codeqlWorkflow, + readRepoFile(t, filepath.Join(".github", "workflows", "benchmark-baseline.yml")), + wantFailure, + ) +} + +func runPairedLauncherAudit(t *testing.T, launcher string, wantFailure bool) string { + t.Helper() + launcherPath := filepath.Join(t.TempDir(), "paired.yml") + if err := os.WriteFile(launcherPath, []byte(launcher), 0o600); err != nil { + t.Fatalf("write paired launcher fixture: %v", err) + } + cmd := exec.Command( + "bash", + "scripts/audit_ci_enforcement.sh", + "--local-only", + "--paired-launcher", + launcherPath, + ) + cmd.Dir = repoRoot(t) + output, err := cmd.CombinedOutput() + if wantFailure { + if err == nil { + t.Fatalf("expected paired launcher audit failure, got success:\n%s", output) + } + return string(output) + } + if err != nil { + t.Fatalf("expected paired launcher audit success, got err=%v output:\n%s", err, output) + } + return string(output) +} + +func runAuditLocalOnlyWithBaseline( + t *testing.T, + workflow string, + codeqlWorkflow string, + baselineWorkflow string, + wantFailure bool, +) string { + t.Helper() + return runAuditFixture( + t, + workflow, + codeqlWorkflow, + baselineWorkflow, + wantFailure, + false, + false, + ) +} + +func runAuditFixture( + t *testing.T, + workflow string, + codeqlWorkflow string, + baselineWorkflow string, + wantFailure bool, + createPairedReference bool, + createPairedThreshold bool, +) string { + t.Helper() + return runAuditFixtureWithTimingValidator( + t, + workflow, + codeqlWorkflow, + baselineWorkflow, + readRepoFile(t, filepath.Join("scripts", "validate_regression_thresholds.py")), + wantFailure, + createPairedReference, + createPairedThreshold, + ) +} + +func runAuditFixtureWithTimingValidator( + t *testing.T, + workflow string, + codeqlWorkflow string, + baselineWorkflow string, + timingValidator string, + wantFailure bool, + createPairedReference bool, + createPairedThreshold bool, +) string { + t.Helper() tmpDir := t.TempDir() workflowPath := filepath.Join(tmpDir, "ci.yml") codeqlWorkflowPath := filepath.Join(tmpDir, "codeql.yml") + baselineWorkflowPath := filepath.Join(tmpDir, "benchmark-baseline.yml") + timingValidatorPath := filepath.Join(tmpDir, "validate_regression_thresholds.py") matrixPath := filepath.Join(tmpDir, "VALIDATION_MATRIX.md") + pairedReferencePath := filepath.Join(tmpDir, "reference-v1.13.json") + pairedThresholdPath := filepath.Join(tmpDir, "threshold-policy-v1.13.json") if err := os.WriteFile(workflowPath, []byte(workflow), 0o600); err != nil { t.Fatalf("write workflow fixture: %v", err) @@ -143,16 +1277,36 @@ func runAuditLocalOnly(t *testing.T, workflow string, codeqlWorkflow string, wan if err := os.WriteFile(codeqlWorkflowPath, []byte(codeqlWorkflow), 0o600); err != nil { t.Fatalf("write codeql workflow fixture: %v", err) } + if err := os.WriteFile(baselineWorkflowPath, []byte(baselineWorkflow), 0o600); err != nil { + t.Fatalf("write benchmark baseline workflow fixture: %v", err) + } + if err := os.WriteFile(timingValidatorPath, []byte(timingValidator), 0o600); err != nil { + t.Fatalf("write timing validator fixture: %v", err) + } if err := os.WriteFile(matrixPath, []byte(readRepoFile(t, "VALIDATION_MATRIX.md")), 0o600); err != nil { t.Fatalf("write validation matrix fixture: %v", err) } + if createPairedReference { + if err := os.WriteFile(pairedReferencePath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write paired reference fixture: %v", err) + } + } + if createPairedThreshold { + if err := os.WriteFile(pairedThresholdPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write paired threshold fixture: %v", err) + } + } cmd := exec.Command("bash", "scripts/audit_ci_enforcement.sh", "--local-only") cmd.Dir = repoRoot(t) cmd.Env = append(os.Environ(), "COLDKEEP_CI_WORKFLOW_FILE="+workflowPath, "COLDKEEP_CODEQL_WORKFLOW_FILE="+codeqlWorkflowPath, + "COLDKEEP_BENCHMARK_BASELINE_WORKFLOW_FILE="+baselineWorkflowPath, + "COLDKEEP_TIMING_VALIDATOR_FILE="+timingValidatorPath, "COLDKEEP_VALIDATION_MATRIX_FILE="+matrixPath, + "COLDKEEP_PAIRED_REFERENCE_MANIFEST_FILE="+pairedReferencePath, + "COLDKEEP_PAIRED_THRESHOLD_POLICY_FILE="+pairedThresholdPath, ) output, err := cmd.CombinedOutput() if wantFailure { diff --git a/scripts/benchmark_gate.py b/scripts/benchmark_gate.py new file mode 100644 index 00000000..7e9294e3 --- /dev/null +++ b/scripts/benchmark_gate.py @@ -0,0 +1,1996 @@ +#!/usr/bin/env python3 +"""Capture and validate statistically bounded Coldkeep benchmark evidence.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import math +import os +import pathlib +import re +import shutil +import statistics +import subprocess +import sys +import time +from decimal import Decimal +from typing import Any, Iterable + +SCHEMA_VERSION = 2 +REPORT_KIND = "benchmark_gate_aggregate" +REVALIDATION_KIND = "benchmark_evidence_contract_revalidation" +INTEGRITY_KIND = "benchmark_integrity" +MANIFEST_KIND = "benchmark_gate_manifest" +FIXTURE_ID = "ci-stable-v1" +FIXTURE_FIELDS = { + "id": FIXTURE_ID, + "seed": 1701, + "large_file_size_bytes": 96 * 1024 * 1024, + "many_small_file_count": 600, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 400, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, +} +INTEGRITY_FIXTURES = { + "ci-paired-w1-v2": { + "id": "ci-paired-w1-v2", + "seed": 1701, + "large_file_size_bytes": 64 * 1024 * 1024, + "many_small_file_count": 400, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 400, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 1, + }, + "ci-paired-w4-v2": { + "id": "ci-paired-w4-v2", + "seed": 1701, + "large_file_size_bytes": 64 * 1024 * 1024, + "many_small_file_count": 400, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 800, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 4, + }, +} +EXPECTED_CASES = [ + "store-large-file", + "store-many-small-files", + "store-mixed-dataset", + "restore-large-file", + "restore-many-files", + "snapshot-creation", + "gc-after-churn", + "stats-inspect", + "verify-system-deep", +] +HARD_ENV_FIELDS = ( + "runner_os", + "runner_arch", + "cpu_count", + "go_version", + "postgres_version", + "database_image_digest", +) +CALIBRATION_IDENTITY_FIELDS = ("source_commit", "binary_sha256", *HARD_ENV_FIELDS) +REQUIRED_PROVENANCE_FIELDS = ( + "source_commit", + "generated_at_utc", + "workflow_run_id", + "workflow_job_id", + "workflow_run_attempt", + "runner_os", + "runner_image", + "runner_arch", + "cpu_count", + "go_version", + "postgres_version", + "database_image_digest", + "binary_sha256", +) +MANIFEST_PROFILES = { + "none-w1": ("none", 1), + "none-w4": ("none", 4), + "zstd-w1": ("zstd", 1), + "zstd-w4": ("zstd", 4), +} +DIAGNOSTIC_SCHEMA_VERSION = 2 +EVIDENCE_POLICY_VERSION = 2 +INT64_MAX = (1 << 63) - 1 +INTEGRITY_SAMPLE_COUNT = 2 +INTEGRITY_COMMAND_TIMEOUT_SECONDS = 600 + +# Outcome E evidence policy. Paths use [] for repeated case/sample records. +# Unknown fields in validated sections fail closed; adding a field requires an +# explicit schema/policy update rather than inheriting an informational policy. +FIELD_POLICY = { + "hard_equal": ( + "raw.schema_version", + "aggregate.schema_version", + "aggregate.evidence_policy_version", + "aggregate.report_kind", + "aggregate.status", + "raw.status", + "raw.command", + "raw.dataset", + "raw.repeat", + "aggregate.provenance.source_commit", + "aggregate.provenance.binary_sha256", + "aggregate.provenance.runner_os", + "aggregate.provenance.runner_arch", + "aggregate.provenance.cpu_count", + "aggregate.provenance.go_version", + "aggregate.provenance.postgres_version", + "aggregate.provenance.database_image_digest", + "profile.codec", + "profile.compression", + "profile.dataset", + "profile.workers", + "profile.pipeline_depth", + "profile.deterministic", + "fixture.*", + "warmup_count", + "sample_count", + "cases[].case", + "cases[].seed", + "cases[].logical_files", + "cases[].logical_bytes", + "cases[].workers_used", + "cases[].diagnostic.active_logical_namespace.*", + "cases[].diagnostic.logical_catalog.*", + "cases[].diagnostic.logical_statuses.*", + "cases[].diagnostic.chunk_graph.*", + "cases[].diagnostic.restored_tree.*", + "cases[].diagnostic.snapshots.*", + "cases[].diagnostic.snapshot_count", + "cases[].diagnostic.gc.*", + "cases[].diagnostic.verification.*", + "cases[].diagnostic.physical.chunk_reference_count", + "cases[].diagnostic.physical.payload_bytes", + "cases[].diagnostic.physical.canonical_sha256", + "operation_totals.*", + "cleanup_totals.*", + ), + "derived_equal": ( + "raw.execution_stats totals recomputed from rows", + "rows[].throughput_mbps", + "rows[].execution_stats.container_append_count", + "rows[].execution_stats.container_open_count", + "rows[].execution_stats.fsync_count", + "rows[].execution_stats.snapshot_metadata_write_count when emitted", + "aggregate.execution_stats.snapshot_metadata_write_count", + "cases[].median_duration_ms", + "cases[].mean_duration_ms", + "cases[].min_duration_ms", + "cases[].max_duration_ms", + "cases[].sample_stddev_ms", + "cases[].mad_ms", + "cases[].mad_ratio_pct", + "cases[].coefficient_of_variation_pct", + "sample_order", + "command_p95_ms", + "manifest.*.sha256", + ), + "bounded_nonnegative": ( + "rows[].execution_stats.container_close_count", + "rows[].execution_stats.io.container_opens", + "rows[].execution_stats.io.container_appends", + "rows[].execution_stats.io.fsyncs", + "rows[].execution_stats.io.bytes_written", + "rows[].execution_stats.io.bytes_read", + ), + "informational": ( + "rows[].duration_ms", + "cases[].sample_durations_ms", + "cases[].operational_samples", + "cases[].operational_counter_distributions", + "cases[].diagnostic_samples[].physical.container_count", + "cases[].diagnostic_samples[].physical.storage_block_count", + "cases[].diagnostic_samples[].physical.legacy_block_count", + "cases[].diagnostic_samples[].physical.container_bytes", + "cases[].diagnostic_samples[].physical_layout_sha256", + "command_durations_ms", + "host_observations", + "provenance.generated_at_utc", + "provenance.workflow_run_id", + "provenance.workflow_job_id", + "provenance.workflow_run_attempt", + "provenance.runner_image", + ), + "excluded_sensitive": ( + "credentials", + "passwords", + "encryption_keys", + "dsns", + "usernames", + "database_names", + "repository_paths", + "temporary_roots", + "sensitive_command_arguments", + "environment_dumps", + "raw_internal_ids", + ), +} + +RAW_ENVELOPE_FIELDS = {"status", "command", "data"} +RAW_DATA_FIELDS = { + "schema_version", "generated_at_utc", "dataset", "repeat", "fixture", + "execution", "execution_stats", "rows", +} +EXECUTION_FIELDS = {"store_folder_workers", "pipeline_depth", "deterministic"} +RAW_ROW_FIELDS = { + "case", "duration_ms", "throughput_mbps", "execution", "execution_stats", + "diagnostic_final_state", +} +IO_COUNTER_FIELDS = { + "container_opens", "container_appends", "fsyncs", "bytes_written", "bytes_read", +} +ROW_EXECUTION_STATS_REQUIRED_FIELDS = { + "total_files", "total_bytes", "workers_used", "container_append_count", + "fsync_count", "container_open_count", "container_close_count", "io", +} +ROW_EXECUTION_STATS_OPTIONAL_FIELDS = {"snapshot_metadata_write_count"} +TOP_EXECUTION_STATS_FIELDS = ROW_EXECUTION_STATS_REQUIRED_FIELDS | { + "snapshot_metadata_write_count", +} +PROVENANCE_FIELDS = set(REQUIRED_PROVENANCE_FIELDS) | {"source_tag"} +PROFILE_FIELDS = {"codec", "compression", "dataset", "workers", "pipeline_depth", "deterministic"} +AGGREGATE_FIELDS = { + "schema_version", "evidence_policy_version", "report_kind", "status", "provenance", + "profile", "fixture", "warmup_count", "sample_count", "sample_order", + "command_durations_ms", "command_p95_ms", "host_observations", "operation_totals", + "cleanup_totals", "cases", +} +AGGREGATE_CASE_FIELDS = { + "case", "seed", "logical_files", "logical_bytes", "workers_used", + "sample_durations_ms", "diagnostic_final_state", "diagnostic_samples", + "operational_samples", "operational_counter_distributions", + "median_duration_ms", "mean_duration_ms", "min_duration_ms", "max_duration_ms", + "sample_stddev_ms", "mad_ms", "mad_ratio_pct", "coefficient_of_variation_pct", + "throughput_mbps", +} +REVALIDATION_FIELDS = { + "schema_version", "evidence_policy_version", "report_kind", "status", + "performance_calibration_status", "profile", "fixture", "warmup_count", + "sample_count", "sample_order", "operation_totals", "cleanup_totals", "cases", +} +SENSITIVE_KEY_PARTS = ( + "password", "credential", "encryption_key", "dsn", "username", "user_name", + "database_name", "db_name", "repository_path", "temporary_root", "temp_root", + "environment_dump", "command_arguments", "command_args", +) +OPERATIONAL_COUNTER_FIELDS = ( + "container_append_count", + "container_open_count", + "container_close_count", + "fsync_count", + "bytes_written", + "bytes_read", + "snapshot_metadata_write_count", +) + + +class GateError(RuntimeError): + """A deterministic benchmark-gate validation failure.""" + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_json_strict(path: pathlib.Path) -> dict[str, Any]: + text = path.read_text(encoding="utf-8") + decoder = json.JSONDecoder(parse_constant=lambda value: (_ for _ in ()).throw( + GateError(f"non-finite JSON value {value!r} in {path}") + )) + try: + value, end = decoder.raw_decode(text) + except (json.JSONDecodeError, GateError) as exc: + raise GateError(f"malformed JSON in {path}: {exc}") from exc + if text[end:].strip(): + raise GateError(f"trailing JSON or content in {path}") + if not isinstance(value, dict): + raise GateError(f"top-level JSON value in {path} must be an object") + return value + + +def write_json(path: pathlib.Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + + +def write_checksums(directory: pathlib.Path) -> None: + entries = [] + for path in sorted(directory.rglob("*")): + if path.is_file() and path.name != "checksums.sha256": + entries.append(f"{sha256_file(path)} {path.relative_to(directory).as_posix()}") + (directory / "checksums.sha256").write_text("\n".join(entries) + "\n", encoding="utf-8") + + +def require_exact_fields(value: Any, expected: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise GateError(f"{label} must be an object") + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + unknown = sorted(actual - expected) + raise GateError(f"{label} fields mismatch: missing={missing} unknown={unknown}") + return value + + +def validate_no_sensitive_evidence(value: Any, label: str = "evidence") -> None: + def visit(item: Any, path: str) -> None: + if isinstance(item, dict): + for key, nested in item.items(): + normalized = str(key).lower().replace("-", "_") + if any(part in normalized for part in SENSITIVE_KEY_PARTS): + raise GateError(f"{label} contains prohibited sensitive field at {path}.{key}") + visit(nested, f"{path}.{key}") + elif isinstance(item, list): + for index, nested in enumerate(item): + visit(nested, f"{path}[{index}]") + elif isinstance(item, str): + lowered = item.lower() + if ( + re.match(r"^[a-z][a-z0-9+.-]*://", item, re.IGNORECASE) + or item.startswith("/") + or re.match(r"^[A-Za-z]:[\\/]", item) + or "coldkeep_bench_" in lowered + or "coldkeep-benchmark-" in lowered + or re.search(r"(?:password|dbname|user)\s*=", lowered) + ): + raise GateError(f"{label} contains prohibited sensitive value at {path}") + + visit(value, label) + + +def write_sanitized_capture(path: pathlib.Path, value: str) -> None: + try: + validate_no_sensitive_evidence({"capture": value}, "captured output") + except GateError: + value = "[captured output omitted: sensitive content]\n" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value, encoding="utf-8") + + +def require_number(value: Any, label: str, *, positive: bool = False) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise GateError(f"{label} must be numeric") + result = float(value) + if not math.isfinite(result): + raise GateError(f"{label} must be finite") + if positive and result <= 0: + raise GateError(f"{label} must be positive") + if not positive and result < 0: + raise GateError(f"{label} must not be negative") + return result + + +def percentile_nearest_rank(values: list[float], percentile: float) -> float: + if not values: + raise GateError("cannot calculate percentile of empty values") + ordered = sorted(values) + index = max(0, math.ceil(percentile * len(ordered)) - 1) + return ordered[index] + + +def summarize(values: Iterable[float]) -> dict[str, float]: + samples = [float(value) for value in values] + if not samples: + raise GateError("cannot summarize an empty sample set") + if any(not math.isfinite(value) or value <= 0 for value in samples): + raise GateError("sample durations must be finite and positive") + median = float(statistics.median(samples)) + mean = float(statistics.mean(samples)) + mad = float(statistics.median(abs(value - median) for value in samples)) + stddev = float(statistics.stdev(samples)) if len(samples) > 1 else 0.0 + return { + "median_duration_ms": median, + "mean_duration_ms": mean, + "min_duration_ms": min(samples), + "max_duration_ms": max(samples), + "sample_stddev_ms": stddev, + "mad_ms": mad, + "mad_ratio_pct": mad / median * 100.0, + "coefficient_of_variation_pct": stddev / mean * 100.0 if mean else 0.0, + } + + +def require_nonnegative_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > INT64_MAX: + raise GateError(f"{label} must be a non-negative signed 64-bit integer") + return value + + +def require_sha256(value: Any, label: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): + raise GateError(f"{label} must be lowercase SHA-256") + return value + + +def validate_diagnostic_final_state(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or value.get("schema_version") != DIAGNOSTIC_SCHEMA_VERSION: + raise GateError(f"{label} diagnostic final state schema mismatch") + expected_keys = { + "schema_version", "active_logical_namespace", "logical_catalog", + "logical_statuses", "chunk_graph", + "restored_tree", "snapshots", "snapshot_count", "gc", "verification", + "physical", "physical_layout_sha256", + } + if set(value) != expected_keys: + raise GateError(f"{label} diagnostic final state fields mismatch") + for section_name in ( + "active_logical_namespace", "logical_catalog", "chunk_graph", "restored_tree", "snapshots" + ): + section = value.get(section_name) + if not isinstance(section, dict) or set(section) != {"count", "total_bytes", "sha256"}: + raise GateError(f"{label} diagnostic {section_name} fields mismatch") + require_nonnegative_integer(section["count"], f"{label} diagnostic {section_name} count") + require_nonnegative_integer( + section["total_bytes"], f"{label} diagnostic {section_name} total_bytes" + ) + require_sha256(section["sha256"], f"{label} diagnostic {section_name} digest") + statuses = value.get("logical_statuses") + if not isinstance(statuses, dict) or set(statuses) != {"completed", "processing", "aborted"}: + raise GateError(f"{label} diagnostic logical status fields mismatch") + for field in statuses: + require_nonnegative_integer(statuses[field], f"{label} diagnostic logical status {field}") + if sum(statuses.values()) != value["logical_catalog"]["count"]: + raise GateError(f"{label} diagnostic logical statuses do not match logical catalog count") + require_nonnegative_integer(value["snapshot_count"], f"{label} diagnostic snapshot_count") + gc_totals = value.get("gc") + expected_gc = { + "total_chunks", "reachable_chunks", "unreachable_chunks", + "logically_reclaimable_bytes", "physically_reclaimable_bytes", + "packed_blocks_live", "packed_blocks_dead", "packed_bytes_live", + "packed_bytes_reclaimable", "retained_dead_bytes", + } + if not isinstance(gc_totals, dict) or set(gc_totals) != expected_gc: + raise GateError(f"{label} diagnostic GC fields mismatch") + for field in gc_totals: + require_nonnegative_integer(gc_totals[field], f"{label} diagnostic GC {field}") + if gc_totals["reachable_chunks"] + gc_totals["unreachable_chunks"] != gc_totals["total_chunks"]: + raise GateError(f"{label} diagnostic GC reachability totals are inconsistent") + verification = value.get("verification") + expected_verification = { + "blocks_checked", "physical_hashes_checked", "compressed_hashes_checked", + "logical_hashes_checked", "compressed_blocks_checked", "physical_file_issues", + "snapshot_membership_rows", "snapshot_reachability_issues", + } + if not isinstance(verification, dict) or set(verification) != expected_verification: + raise GateError(f"{label} diagnostic verification fields mismatch") + for field in verification: + require_nonnegative_integer( + verification[field], f"{label} diagnostic verification {field}" + ) + if verification["physical_file_issues"] != 0 or verification["snapshot_reachability_issues"] != 0: + raise GateError(f"{label} diagnostic verification reports integrity issues") + if verification["snapshot_membership_rows"] != value["snapshots"]["count"]: + raise GateError(f"{label} diagnostic snapshot membership totals are inconsistent") + physical = value.get("physical") + expected_physical = { + "container_count", "storage_block_count", "legacy_block_count", + "chunk_reference_count", "payload_bytes", "container_bytes", "canonical_sha256", + } + if not isinstance(physical, dict) or set(physical) != expected_physical: + raise GateError(f"{label} diagnostic physical fields mismatch") + for field in expected_physical - {"canonical_sha256"}: + require_nonnegative_integer(physical[field], f"{label} diagnostic physical {field}") + require_sha256(physical["canonical_sha256"], f"{label} diagnostic canonical physical digest") + require_sha256(value["physical_layout_sha256"], f"{label} diagnostic physical layout digest") + validate_no_sensitive_evidence(value, label) + return value + + +def hard_final_state(row: dict[str, Any]) -> dict[str, Any]: + state = validate_diagnostic_final_state( + row.get("diagnostic_final_state"), f"case {row.get('case')!r}" + ) + physical = state["physical"] + return { + "active_logical_namespace": state["active_logical_namespace"], + "logical_catalog": state["logical_catalog"], + "logical_statuses": state["logical_statuses"], + "chunk_graph": state["chunk_graph"], + "restored_tree": state["restored_tree"], + "snapshots": state["snapshots"], + "snapshot_count": state["snapshot_count"], + "gc": state["gc"], + "verification": state["verification"], + "physical_content": { + "chunk_reference_count": physical["chunk_reference_count"], + "payload_bytes": physical["payload_bytes"], + "canonical_sha256": physical["canonical_sha256"], + }, + } + + +def validate_execution(value: Any, *, workers: int, label: str) -> dict[str, Any]: + value = require_exact_fields(value, EXECUTION_FIELDS, label) + if ( + value["store_folder_workers"] != workers + or value["pipeline_depth"] != 1 + or value["deterministic"] is not True + ): + raise GateError(f"{label} policy mismatch") + return value + + +def validate_operational_counters(row: dict[str, Any], *, workers: int) -> dict[str, int]: + case_name = row.get("case") + stats = row.get("execution_stats") + if not isinstance(stats, dict): + raise GateError(f"{case_name} execution_stats must be an object") + fields = set(stats) + if not ROW_EXECUTION_STATS_REQUIRED_FIELDS <= fields: + missing = sorted(ROW_EXECUTION_STATS_REQUIRED_FIELDS - fields) + raise GateError(f"{case_name} execution_stats missing mandatory counters: {missing}") + unknown = fields - ROW_EXECUTION_STATS_REQUIRED_FIELDS - ROW_EXECUTION_STATS_OPTIONAL_FIELDS + if unknown: + raise GateError(f"{case_name} execution_stats has unknown fields: {sorted(unknown)}") + io_stats = require_exact_fields(stats["io"], IO_COUNTER_FIELDS, f"{case_name} I/O counters") + + logical_files = require_nonnegative_integer(stats["total_files"], f"{case_name} logical files") + logical_bytes = require_nonnegative_integer(stats["total_bytes"], f"{case_name} logical bytes") + workers_used = require_nonnegative_integer(stats["workers_used"], f"{case_name} workers_used") + if logical_files <= 0 or logical_bytes <= 0: + raise GateError(f"{case_name} logical totals must be positive") + if workers_used != workers: + raise GateError(f"{case_name} workers_used mismatch") + + outer_append = require_nonnegative_integer( + stats["container_append_count"], f"{case_name} container_append_count" + ) + outer_open = require_nonnegative_integer( + stats["container_open_count"], f"{case_name} container_open_count" + ) + outer_close = require_nonnegative_integer( + stats["container_close_count"], f"{case_name} container_close_count" + ) + outer_fsync = require_nonnegative_integer(stats["fsync_count"], f"{case_name} fsync_count") + io_values = { + field: require_nonnegative_integer(io_stats[field], f"{case_name} I/O counter {field}") + for field in IO_COUNTER_FIELDS + } + snapshot_writes = require_nonnegative_integer( + stats.get("snapshot_metadata_write_count", 0), + f"{case_name} snapshot_metadata_write_count", + ) + + if outer_append != io_values["container_appends"]: + raise GateError(f"{case_name} duplicated container append counters differ") + if outer_open != io_values["container_opens"]: + raise GateError(f"{case_name} duplicated container open counters differ") + if outer_fsync != io_values["fsyncs"]: + raise GateError(f"{case_name} duplicated fsync counters differ") + if outer_open != outer_close: + raise GateError(f"{case_name} container open/close counters are unbalanced") + if outer_append > 0 and ( + outer_open == 0 or outer_fsync == 0 or io_values["bytes_written"] == 0 + ): + raise GateError(f"{case_name} append counters contradict execution I/O") + if io_values["bytes_read"] > 0 and outer_open == 0: + raise GateError(f"{case_name} read counters contradict container opens") + if snapshot_writes > 0 and case_name not in {"snapshot-creation", "gc-after-churn"}: + raise GateError(f"{case_name} snapshot writes contradict the operation type") + + return { + "container_append_count": outer_append, + "container_open_count": outer_open, + "container_close_count": outer_close, + "fsync_count": outer_fsync, + "bytes_written": io_values["bytes_written"], + "bytes_read": io_values["bytes_read"], + "snapshot_metadata_write_count": snapshot_writes, + } + + +def hard_row_contract(row: dict[str, Any], *, workers: int) -> dict[str, Any]: + validate_execution(row.get("execution"), workers=workers, label=f"{row.get('case')} execution") + validate_operational_counters(row, workers=workers) + stats = row["execution_stats"] + return { + "case": row["case"], + "execution": row["execution"], + "logical_files": stats["total_files"], + "logical_bytes": stats["total_bytes"], + "workers_used": stats["workers_used"], + "diagnostic_final_state": hard_final_state(row), + } + + +def hard_aggregate_case_contract(case: dict[str, Any]) -> dict[str, Any]: + """Fields that must remain exact when aggregate cases are compared.""" + return { + "case": case["case"], + "seed": case["seed"], + "logical_files": case["logical_files"], + "logical_bytes": case["logical_bytes"], + "workers_used": case["workers_used"], + "diagnostic_final_state": hard_final_state(case), + } + + +def summarize_operational_counters(samples: list[dict[str, int]]) -> dict[str, dict[str, Any]]: + if not samples: + raise GateError("cannot summarize empty operational counter samples") + return { + field: { + "min": min(sample[field] for sample in samples), + "max": max(sample[field] for sample in samples), + "values": sorted({sample[field] for sample in samples}), + } + for field in OPERATIONAL_COUNTER_FIELDS + } + + +def validate_operational_distributions( + samples: Any, + distributions: Any, + *, + sample_count: int, + label: str, +) -> None: + if not isinstance(samples, list) or len(samples) != sample_count: + raise GateError(f"{label} operational sample count mismatch") + normalized: list[dict[str, int]] = [] + for index, sample in enumerate(samples): + sample = require_exact_fields(sample, set(OPERATIONAL_COUNTER_FIELDS), f"{label} operational sample {index + 1}") + normalized.append({ + field: require_nonnegative_integer(sample[field], f"{label} operational {field}") + for field in OPERATIONAL_COUNTER_FIELDS + }) + if normalized[-1]["container_open_count"] != normalized[-1]["container_close_count"]: + raise GateError(f"{label} operational sample {index + 1} is unbalanced") + if distributions != summarize_operational_counters(normalized): + raise GateError(f"{label} operational counter distributions are inconsistent") + + +def validate_top_execution_stats(value: Any, rows: list[dict[str, Any]], *, workers: int) -> None: + value = require_exact_fields(value, TOP_EXECUTION_STATS_FIELDS, "raw aggregate execution_stats") + io_stats = require_exact_fields(value["io"], IO_COUNTER_FIELDS, "raw aggregate I/O counters") + expected = { + "total_files": sum(row["execution_stats"]["total_files"] for row in rows), + "total_bytes": sum(row["execution_stats"]["total_bytes"] for row in rows), + "workers_used": max(row["execution_stats"]["workers_used"] for row in rows), + "container_append_count": sum(row["execution_stats"]["container_append_count"] for row in rows), + "fsync_count": sum(row["execution_stats"]["fsync_count"] for row in rows), + "container_open_count": sum(row["execution_stats"]["container_open_count"] for row in rows), + "container_close_count": sum(row["execution_stats"]["container_close_count"] for row in rows), + "snapshot_metadata_write_count": sum( + row["execution_stats"].get("snapshot_metadata_write_count", 0) for row in rows + ), + } + for field, expected_value in expected.items(): + actual = require_nonnegative_integer(value[field], f"raw aggregate {field}") + if actual != expected_value: + raise GateError(f"raw aggregate {field} does not match case rows") + if value["workers_used"] != workers: + raise GateError("raw aggregate workers_used mismatch") + expected_io = { + field: sum(row["execution_stats"]["io"][field] for row in rows) + for field in IO_COUNTER_FIELDS + } + for field, expected_value in expected_io.items(): + actual = require_nonnegative_integer(io_stats[field], f"raw aggregate I/O {field}") + if actual != expected_value: + raise GateError(f"raw aggregate I/O {field} does not match case rows") + + +def fixture_fields(dataset: str) -> dict[str, Any]: + if dataset == FIXTURE_ID: + return FIXTURE_FIELDS + configured = INTEGRITY_FIXTURES.get(dataset) + if configured is None: + raise GateError(f"unsupported benchmark fixture {dataset!r}") + return {key: value for key, value in configured.items() if key != "workers"} + + +def _require_fixture_dataset(value: Any) -> str: + fixture_fields(value) + if not isinstance(value, str): + raise GateError(f"unsupported benchmark fixture {value!r}") + return value + + +def validate_fixture(fixture: Any, *, dataset: str = FIXTURE_ID) -> list[dict[str, Any]]: + expected_fields = fixture_fields(dataset) + fixture = require_exact_fields( + fixture, + set(expected_fields) | {"ordered_cases"}, + "fixture", + ) + for field, expected in expected_fields.items(): + if fixture.get(field) != expected: + raise GateError(f"fixture field {field!r} does not match {dataset}") + ordered = fixture.get("ordered_cases") + if not isinstance(ordered, list) or len(ordered) != len(EXPECTED_CASES): + raise GateError("fixture ordered case count mismatch") + for index, (descriptor, expected_name) in enumerate(zip(ordered, EXPECTED_CASES)): + if not isinstance(descriptor, dict): + raise GateError(f"fixture case at index {index} must be an object") + require_exact_fields(descriptor, {"name", "seed"}, f"fixture case at index {index}") + expected_seed = 1712 + index * 10 + if descriptor.get("name") != expected_name or descriptor.get("seed") != expected_seed: + raise GateError(f"fixture case descriptor mismatch at index {index}") + return ordered + + +def validate_provenance(value: Any) -> dict[str, Any]: + value = require_exact_fields(value, PROVENANCE_FIELDS, "aggregate provenance") + for field in REQUIRED_PROVENANCE_FIELDS: + if value.get(field) in (None, "", "unknown"): + raise GateError(f"aggregate provenance field {field!r} is missing") + if not re.fullmatch(r"[0-9a-f]{40}", str(value["source_commit"])): + raise GateError("aggregate source_commit must be a full lowercase commit SHA") + if not re.fullmatch(r"[0-9a-f]{64}", str(value["binary_sha256"])): + raise GateError("aggregate binary_sha256 must be lowercase SHA-256") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", str(value["database_image_digest"])): + raise GateError("aggregate database_image_digest must be a sha256 digest") + if ( + isinstance(value["cpu_count"], bool) + or not isinstance(value["cpu_count"], int) + or value["cpu_count"] <= 0 + ): + raise GateError("aggregate cpu_count must be a positive integer") + generated = str(value["generated_at_utc"]) + if not generated.endswith("Z"): + raise GateError("aggregate generated_at_utc must be UTC") + try: + dt.datetime.fromisoformat(generated[:-1] + "+00:00") + except ValueError as exc: + raise GateError("aggregate generated_at_utc must be RFC3339") from exc + validate_no_sensitive_evidence(value, "aggregate provenance") + return value + + +def validate_raw_report( + envelope: dict[str, Any], + *, + workers: int, + compression: str, + dataset: str = FIXTURE_ID, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + require_exact_fields(envelope, RAW_ENVELOPE_FIELDS, "raw report envelope") + if envelope.get("status") != "ok" or envelope.get("command") != "benchmark": + raise GateError("raw report must be a successful benchmark envelope") + data = require_exact_fields(envelope.get("data"), RAW_DATA_FIELDS, "raw report data") + if data.get("schema_version") != SCHEMA_VERSION: + raise GateError(f"raw report schema must be {SCHEMA_VERSION}") + if data.get("dataset") != dataset or data.get("repeat") != 1: + raise GateError("raw report has the wrong dataset or repeat count") + + execution = validate_execution(data.get("execution"), workers=workers, label="raw report execution") + + validate_fixture(data.get("fixture"), dataset=dataset) + + rows = data.get("rows") + if not isinstance(rows, list) or not rows: + raise GateError("raw report rows must be a non-empty array") + names = [row.get("case") for row in rows if isinstance(row, dict)] + if names != EXPECTED_CASES: + raise GateError("raw report case set/order mismatch") + if len(set(names)) != len(names): + raise GateError("raw report contains duplicate cases") + + for row in rows: + if not isinstance(row, dict): + raise GateError("raw report row must be an object") + require_exact_fields(row, RAW_ROW_FIELDS, f"raw report row {row.get('case')!r}") + duration = require_number(row.get("duration_ms"), f"{row.get('case')} duration", positive=True) + throughput = require_number( + row.get("throughput_mbps"), + f"{row.get('case')} throughput", + positive=True, + ) + row_execution = validate_execution( + row.get("execution"), workers=workers, label=f"{row.get('case')} execution" + ) + if row_execution != execution: + raise GateError(f"{row.get('case')} execution policy mismatch") + validate_operational_counters(row, workers=workers) + stats = row["execution_stats"] + logical_bytes = stats["total_bytes"] + expected_throughput = logical_bytes / (1024.0 * 1024.0) / (duration / 1000.0) + if not math.isclose(throughput, expected_throughput, rel_tol=1e-12, abs_tol=1e-12): + raise GateError(f"{row.get('case')} derived throughput is inconsistent") + hard_final_state(row) + + validate_top_execution_stats(data.get("execution_stats"), rows, workers=workers) + + # Compression is supplied by the controlled environment rather than the v2 + # raw payload. Recording it here makes that ownership explicit. + if compression not in {"none", "zstd"}: + raise GateError(f"unsupported compression profile {compression!r}") + validate_no_sensitive_evidence(envelope, "raw report") + return data, rows + + +def command_output(command: list[str]) -> str: + completed = subprocess.run(command, check=True, text=True, capture_output=True) + return completed.stdout.strip() + + +def git_value(args: list[str], default: str = "unknown") -> str: + try: + return command_output(["git", *args]) + except (OSError, subprocess.CalledProcessError): + return default + + +def host_load() -> dict[str, Any]: + load = os.getloadavg() if hasattr(os, "getloadavg") else (0.0, 0.0, 0.0) + disk = shutil.disk_usage(pathlib.Path.cwd()) + return { + "load_1m": load[0], + "load_5m": load[1], + "load_15m": load[2], + "free_disk_bytes": disk.free, + } + + +def provenance(args: argparse.Namespace, binary_hash: str) -> dict[str, Any]: + return { + "source_commit": args.source_commit or os.environ.get("GITHUB_SHA") or git_value(["rev-parse", "HEAD"]), + "source_tag": args.source_tag or None, + "generated_at_utc": utc_now(), + "workflow_run_id": os.environ.get("GITHUB_RUN_ID", "local"), + "workflow_job_id": os.environ.get("GITHUB_JOB", "local"), + "workflow_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", "local"), + "runner_os": os.environ.get("RUNNER_OS", sys.platform), + "runner_image": os.environ.get("ImageVersion", "local"), + "runner_arch": os.environ.get("RUNNER_ARCH", os.uname().machine if hasattr(os, "uname") else "unknown"), + "cpu_count": os.cpu_count() or 0, + "go_version": args.go_version or command_output(["go", "version"]), + "postgres_version": args.postgres_version, + "database_image_digest": args.database_image_digest, + "binary_sha256": binary_hash, + } + + +def _benchmark_sample_command(args: argparse.Namespace) -> list[str]: + return [ + str(args.binary), + "benchmark", + "run", + "--dataset", + args.dataset, + "--workers", + str(args.workers), + "--repeat", + "1", + "--output", + "json", + ] + + +def capture_sample( + args: argparse.Namespace, + output_path: pathlib.Path, + expected_binary_hash: str, +) -> tuple[dict[str, Any], float, dict[str, Any]]: + if sha256_file(args.binary) != expected_binary_hash: + raise GateError("benchmark binary hash changed during sampling") + before = host_load() + started = time.monotonic() + try: + completed = subprocess.run( + _benchmark_sample_command(args), + check=False, + text=True, + capture_output=True, + env={**os.environ, "COLDKEEP_COMPRESSION": args.compression}, + timeout=getattr(args, "command_timeout_seconds", None), + ) + except subprocess.TimeoutExpired as exc: + output_path.parent.mkdir(parents=True, exist_ok=True) + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr = exc.stderr if isinstance(exc.stderr, str) else "" + output_path.write_text(stdout, encoding="utf-8") + write_sanitized_capture(output_path.with_suffix(".stderr"), stderr) + raise GateError("benchmark command timeout") from exc + elapsed_ms = (time.monotonic() - started) * 1000.0 + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(completed.stdout, encoding="utf-8") + write_sanitized_capture(output_path.with_suffix(".stderr"), completed.stderr) + if completed.returncode != 0: + raise GateError(f"benchmark sample failed with exit {completed.returncode}") + envelope = load_json_strict(output_path) + validate_raw_report( + envelope, + workers=args.workers, + compression=args.compression, + dataset=args.dataset, + ) + return envelope, elapsed_ms, {"before": before, "after": host_load()} + + +def build_contract_cases( + raw_reports: list[dict[str, Any]], + *, + workers: int, + compression: str, + dataset: str = FIXTURE_ID, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + if not raw_reports: + raise GateError("cannot aggregate an empty raw report set") + first_data, first_rows = validate_raw_report( + raw_reports[0], workers=workers, compression=compression, dataset=dataset + ) + fixture_cases = first_data["fixture"]["ordered_cases"] + cases: list[dict[str, Any]] = [] + for case_index, first_row in enumerate(first_rows): + case_name = first_row["case"] + expected_hard = hard_row_contract(first_row, workers=workers) + first_diagnostic = first_row["diagnostic_final_state"] + expected_restored_files = ( + first_row["execution_stats"]["total_files"] + if case_name in {"restore-large-file", "restore-many-files"} + else 0 + ) + if first_diagnostic["restored_tree"]["count"] != expected_restored_files: + raise GateError(f"{case_name} restored file total mismatch") + durations: list[float] = [] + diagnostic_samples: list[dict[str, Any]] = [] + operational_samples: list[dict[str, int]] = [] + for sample_index, envelope in enumerate(raw_reports): + data, rows = validate_raw_report( + envelope, workers=workers, compression=compression, dataset=dataset + ) + if data["fixture"] != first_data["fixture"] or data["execution"] != first_data["execution"]: + raise GateError(f"fixture/profile changed in sample {sample_index + 1}") + row = rows[case_index] + if row["case"] != case_name: + raise GateError(f"case order changed in sample {sample_index + 1}") + if hard_row_contract(row, workers=workers) != expected_hard: + raise GateError( + f"hard evidence changed for {case_name} in sample {sample_index + 1}" + ) + durations.append(float(row["duration_ms"])) + diagnostic_samples.append(row["diagnostic_final_state"]) + operational_samples.append(validate_operational_counters(row, workers=workers)) + + summary = summarize(durations) + logical_bytes = first_row["execution_stats"]["total_bytes"] + summary["throughput_mbps"] = ( + logical_bytes / (1024.0 * 1024.0) / (summary["median_duration_ms"] / 1000.0) + ) + cases.append( + { + "case": case_name, + "seed": fixture_cases[case_index]["seed"], + "logical_files": first_row["execution_stats"]["total_files"], + "logical_bytes": logical_bytes, + "workers_used": first_row["execution_stats"]["workers_used"], + "sample_durations_ms": durations, + "diagnostic_final_state": diagnostic_samples[0], + "diagnostic_samples": diagnostic_samples, + "operational_samples": operational_samples, + "operational_counter_distributions": summarize_operational_counters( + operational_samples + ), + **summary, + } + ) + return first_data, cases + + +def operation_totals(sample_count: int) -> dict[str, int]: + total = sample_count * len(EXPECTED_CASES) + return {"success": total, "failure": 0, "skipped": 0} + + +def cleanup_totals(sample_count: int) -> dict[str, int]: + total = sample_count * len(EXPECTED_CASES) + return { + "attempted": total, + "succeeded": total, + "failed": 0, + "leaked_databases": 0, + "leaked_processes": 0, + "leaked_temporary_resources": 0, + } + + +def validate_host_observations(value: Any, *, sample_count: int) -> None: + if not isinstance(value, list) or len(value) != sample_count: + raise GateError("aggregate host observation count mismatch") + host_fields = {"load_1m", "load_5m", "load_15m", "free_disk_bytes"} + for sample_index, observation in enumerate(value): + observation = require_exact_fields( + observation, + {"before", "after"}, + f"host observation {sample_index + 1}", + ) + for point in ("before", "after"): + values = require_exact_fields( + observation[point], + host_fields, + f"host observation {sample_index + 1} {point}", + ) + for field in host_fields: + require_number(values[field], f"host observation {sample_index + 1} {point} {field}") + + +def validate_operation_and_cleanup_totals(report: dict[str, Any], *, sample_count: int) -> None: + expected_operations = operation_totals(sample_count) + operations = require_exact_fields( + report.get("operation_totals"), set(expected_operations), "aggregate operation_totals" + ) + for field, expected in expected_operations.items(): + actual = require_nonnegative_integer(operations[field], f"operation total {field}") + if actual != expected: + raise GateError(f"operation total {field} mismatch") + + expected_cleanup = cleanup_totals(sample_count) + cleanup = require_exact_fields( + report.get("cleanup_totals"), set(expected_cleanup), "aggregate cleanup_totals" + ) + for field, expected in expected_cleanup.items(): + actual = require_nonnegative_integer(cleanup[field], f"cleanup total {field}") + if actual != expected: + raise GateError(f"cleanup total {field} mismatch") + + +def sample_command(args: argparse.Namespace) -> int: + if args.dataset != FIXTURE_ID: + raise GateError(f"gate sampling requires dataset {FIXTURE_ID!r}") + if args.warmups < 0 or args.samples <= 0: + raise GateError("warmups must be non-negative and samples must be positive") + if not args.binary.is_file(): + raise GateError(f"binary does not exist: {args.binary}") + args.binary = args.binary.resolve() + if os.environ.get("COLDKEEP_CODEC") != "aes-gcm": + raise GateError("gate sampling requires COLDKEEP_CODEC=aes-gcm") + for name in ("DB_HOST", "DB_PORT", "DB_USER", "DB_PASSWORD", "DB_NAME", "DB_SSLMODE"): + if not os.environ.get(name): + raise GateError(f"gate sampling requires {name}") + args.output_dir.parent.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(args.output_dir.parent).free < args.minimum_free_disk_bytes: + raise GateError("insufficient free disk for benchmark sampling") + + args.output_dir.mkdir(parents=True, exist_ok=False) + binary_hash = sha256_file(args.binary) + raw_reports: list[dict[str, Any]] = [] + command_durations: list[float] = [] + loads: list[dict[str, Any]] = [] + + for index in range(args.warmups): + capture_sample( + args, + args.output_dir / "raw" / f"warmup-{index + 1:02d}.json", + binary_hash, + ) + for index in range(args.samples): + envelope, elapsed_ms, load = capture_sample( + args, + args.output_dir / "raw" / f"sample-{index + 1:02d}.json", + binary_hash, + ) + raw_reports.append(envelope) + command_durations.append(elapsed_ms) + loads.append(load) + + first_data, cases = build_contract_cases( + raw_reports, + workers=args.workers, + compression=args.compression, + dataset=args.dataset, + ) + + aggregate = { + "schema_version": SCHEMA_VERSION, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": REPORT_KIND, + "status": "ok", + "provenance": provenance(args, binary_hash), + "profile": { + "codec": os.environ.get("COLDKEEP_CODEC", ""), + "compression": args.compression, + "dataset": args.dataset, + "workers": args.workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "fixture": first_data["fixture"], + "warmup_count": args.warmups, + "sample_count": args.samples, + "sample_order": list(range(1, args.samples + 1)), + "command_durations_ms": command_durations, + "command_p95_ms": percentile_nearest_rank(command_durations, 0.95), + "host_observations": loads, + "operation_totals": operation_totals(args.samples), + "cleanup_totals": cleanup_totals(args.samples), + "cases": cases, + } + write_json(args.output_dir / "aggregate.json", aggregate) + print(args.output_dir / "aggregate.json") + return 0 + + +def validate_integrity_report(report: dict[str, Any]) -> None: + report = require_exact_fields( + report, + { + "schema_version", "evidence_policy_version", "report_kind", "status", + "classification", "authority", "profile", "expected_sample_count", + "completed_sample_count", "completed_prefix", "active_invocation", + "incomplete_invocation", "aggregate_file", "hard_state", "counters", + "cleanup", "failure", + }, + "benchmark integrity report", + ) + if ( + report["schema_version"] != 1 + or report["evidence_policy_version"] != EVIDENCE_POLICY_VERSION + or report["report_kind"] != INTEGRITY_KIND + ): + raise GateError("benchmark integrity report identity mismatch") + require_exact_fields( + report["authority"], + {"integrity_authority", "performance_authority"}, + "benchmark integrity authority", + ) + if report["authority"] != {"integrity_authority": True, "performance_authority": False}: + raise GateError("benchmark integrity authority mismatch") + profile = require_exact_fields( + report["profile"], {"compression", "workers", "dataset"}, "benchmark integrity profile" + ) + configured = INTEGRITY_FIXTURES.get(profile["dataset"]) + if ( + configured is None + or configured["workers"] != profile["workers"] + or profile["compression"] not in {"none", "zstd"} + ): + raise GateError("benchmark integrity profile mismatch") + expected = require_nonnegative_integer(report["expected_sample_count"], "expected sample count") + completed = require_nonnegative_integer(report["completed_sample_count"], "completed sample count") + if ( + expected != INTEGRITY_SAMPLE_COUNT + or completed > expected + or report["completed_prefix"] != [f"raw/sample-{index:02d}.json" for index in range(1, completed + 1)] + ): + raise GateError("benchmark integrity sample inventory mismatch") + if report["status"] == "complete": + if ( + report["classification"] != "BENCHMARK_INTEGRITY_PASS" + or completed != expected + or report["aggregate_file"] != "aggregate.json" + or report["hard_state"] != "equal" + or report["counters"] != "valid" + or report["cleanup"] != "complete" + or report["failure"] is not None + or report["active_invocation"] is not None + or report["incomplete_invocation"] is not None + ): + raise GateError("benchmark integrity success claims mismatch") + elif report["status"] == "failed": + if ( + report["classification"] != "BENCHMARK_INTEGRITY_FAILURE" + or report["aggregate_file"] is not None + or report["hard_state"] not in {"not_evaluated", "prefix_valid"} + or report["counters"] not in {"not_evaluated", "prefix_valid"} + or report["cleanup"] not in {"not_verified", "complete"} + or report["failure"] not in { + "command_timeout", "contract_or_command_failure", "infrastructure_failure" + } + or report["active_invocation"] is not None + or report["incomplete_invocation"] != ( + None + if completed == expected + else { + "sample_index": completed + 1, + "raw_file": f"raw/sample-{completed + 1:02d}.json", + "stderr_file": f"raw/sample-{completed + 1:02d}.stderr", + } + ) + ): + raise GateError("benchmark integrity failure claims mismatch") + else: + raise GateError("benchmark integrity status mismatch") + validate_no_sensitive_evidence(report, "benchmark integrity report") + + +def integrity_command(args: argparse.Namespace) -> int: + configured = INTEGRITY_FIXTURES.get(args.dataset) + if configured is None or configured["workers"] != args.workers: + raise GateError("integrity dataset and worker profile mismatch") + if args.command_timeout_seconds != INTEGRITY_COMMAND_TIMEOUT_SECONDS: + raise GateError("integrity command timeout must be 600 seconds") + if not args.binary.is_file(): + raise GateError("integrity binary does not exist") + if args.output_dir.exists(): + raise GateError("integrity output directory must not exist") + if os.environ.get("COLDKEEP_CODEC") != "aes-gcm": + raise GateError("integrity sampling requires COLDKEEP_CODEC=aes-gcm") + for name in ("DB_HOST", "DB_PORT", "DB_USER", "DB_PASSWORD", "DB_NAME", "DB_SSLMODE"): + if not os.environ.get(name): + raise GateError(f"integrity sampling requires {name}") + + args.binary = args.binary.resolve() + args.output_dir.parent.mkdir(parents=True, exist_ok=True) + args.output_dir.mkdir() + binary_hash = sha256_file(args.binary) + raw_reports: list[dict[str, Any]] = [] + command_durations: list[float] = [] + loads: list[dict[str, Any]] = [] + failure: str | None = None + try: + for index in range(INTEGRITY_SAMPLE_COUNT): + envelope, elapsed_ms, load = capture_sample( + args, + args.output_dir / "raw" / f"sample-{index + 1:02d}.json", + binary_hash, + ) + raw_reports.append(envelope) + command_durations.append(elapsed_ms) + loads.append(load) + first_data, cases = build_contract_cases( + raw_reports, + workers=args.workers, + compression=args.compression, + dataset=args.dataset, + ) + aggregate = { + "schema_version": SCHEMA_VERSION, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": REPORT_KIND, + "status": "ok", + "provenance": provenance(args, binary_hash), + "profile": { + "codec": "aes-gcm", + "compression": args.compression, + "dataset": args.dataset, + "workers": args.workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "fixture": first_data["fixture"], + "warmup_count": 0, + "sample_count": INTEGRITY_SAMPLE_COUNT, + "sample_order": [1, 2], + "command_durations_ms": command_durations, + "command_p95_ms": percentile_nearest_rank(command_durations, 0.95), + "host_observations": loads, + "operation_totals": operation_totals(INTEGRITY_SAMPLE_COUNT), + "cleanup_totals": cleanup_totals(INTEGRITY_SAMPLE_COUNT), + "cases": cases, + } + validate_aggregate(aggregate, require_gate_count=False) + write_json(args.output_dir / "aggregate.json", aggregate) + report = { + "schema_version": 1, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": INTEGRITY_KIND, + "status": "complete", + "classification": "BENCHMARK_INTEGRITY_PASS", + "authority": {"integrity_authority": True, "performance_authority": False}, + "profile": {"compression": args.compression, "workers": args.workers, "dataset": args.dataset}, + "expected_sample_count": INTEGRITY_SAMPLE_COUNT, + "completed_sample_count": len(raw_reports), + "completed_prefix": [f"raw/sample-{index:02d}.json" for index in range(1, len(raw_reports) + 1)], + "active_invocation": None, + "incomplete_invocation": None, + "aggregate_file": "aggregate.json", + "hard_state": "equal", + "counters": "valid", + "cleanup": "complete", + "failure": None, + } + except subprocess.TimeoutExpired: + failure = "command_timeout" + except GateError as exc: + failure = "command_timeout" if "timeout" in str(exc).lower() else "contract_or_command_failure" + except (OSError, subprocess.CalledProcessError): + failure = "infrastructure_failure" + + if failure is not None: + prefix_valid = bool(raw_reports) + report = { + "schema_version": 1, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": INTEGRITY_KIND, + "status": "failed", + "classification": "BENCHMARK_INTEGRITY_FAILURE", + "authority": {"integrity_authority": True, "performance_authority": False}, + "profile": {"compression": args.compression, "workers": args.workers, "dataset": args.dataset}, + "expected_sample_count": INTEGRITY_SAMPLE_COUNT, + "completed_sample_count": len(raw_reports), + "completed_prefix": [f"raw/sample-{index:02d}.json" for index in range(1, len(raw_reports) + 1)], + "active_invocation": None, + "incomplete_invocation": ( + None + if len(raw_reports) == INTEGRITY_SAMPLE_COUNT + else { + "sample_index": len(raw_reports) + 1, + "raw_file": f"raw/sample-{len(raw_reports) + 1:02d}.json", + "stderr_file": f"raw/sample-{len(raw_reports) + 1:02d}.stderr", + } + ), + "aggregate_file": None, + "hard_state": "prefix_valid" if prefix_valid else "not_evaluated", + "counters": "prefix_valid" if prefix_valid else "not_evaluated", + "cleanup": "not_verified", + "failure": failure, + } + validate_integrity_report(report) + write_json(args.output_dir / "benchmark-integrity.json", report) + write_checksums(args.output_dir) + print(json.dumps({"classification": report["classification"]})) + return 0 if report["classification"] == "BENCHMARK_INTEGRITY_PASS" else 2 + + +def validate_aggregate(report: dict[str, Any], *, require_gate_count: bool) -> None: + require_exact_fields(report, AGGREGATE_FIELDS, "aggregate") + if ( + report.get("schema_version") != SCHEMA_VERSION + or report.get("evidence_policy_version") != EVIDENCE_POLICY_VERSION + or report.get("report_kind") != REPORT_KIND + or report.get("status") != "ok" + ): + raise GateError("aggregate schema/policy/report kind/status mismatch") + sample_count = require_nonnegative_integer(report.get("sample_count"), "aggregate sample_count") + warmup_count = require_nonnegative_integer(report.get("warmup_count"), "aggregate warmup_count") + if sample_count <= 0: + raise GateError("aggregate sample_count must be a positive integer") + if require_gate_count and (sample_count != 5 or warmup_count != 1): + raise GateError("required gate expects one warmup and five samples") + if report.get("sample_order") != list(range(1, sample_count + 1)): + raise GateError("aggregate sample order mismatch") + profile = require_exact_fields(report.get("profile"), PROFILE_FIELDS, "aggregate profile") + profile_dataset = _require_fixture_dataset(profile.get("dataset")) + fixture_cases = validate_fixture(report.get("fixture"), dataset=profile_dataset) + validate_provenance(report.get("provenance")) + if ( + profile.get("dataset") not in {FIXTURE_ID, *INTEGRITY_FIXTURES} + or profile.get("codec") != "aes-gcm" + or profile.get("compression") not in {"none", "zstd"} + or profile.get("workers") not in {1, 4} + or profile.get("pipeline_depth") != 1 + or profile.get("deterministic") is not True + ): + raise GateError("aggregate fixture/profile contract mismatch") + command_durations = report.get("command_durations_ms") + if not isinstance(command_durations, list) or len(command_durations) != sample_count: + raise GateError("aggregate command duration count mismatch") + command_durations = [ + require_number(value, "aggregate command duration", positive=True) + for value in command_durations + ] + expected_p95 = percentile_nearest_rank(command_durations, 0.95) + actual_p95 = require_number(report.get("command_p95_ms"), "aggregate command p95", positive=True) + if not math.isclose(actual_p95, expected_p95, rel_tol=1e-12, abs_tol=1e-9): + raise GateError("aggregate command p95 is inconsistent") + validate_host_observations(report.get("host_observations"), sample_count=sample_count) + validate_operation_and_cleanup_totals(report, sample_count=sample_count) + + cases = report.get("cases") + if not isinstance(cases, list) or not cases: + raise GateError("aggregate cases must be non-empty") + names = [case.get("case") for case in cases if isinstance(case, dict)] + if names != EXPECTED_CASES or len(set(names)) != len(names): + raise GateError("aggregate case set/order mismatch") + expected_execution = { + "store_folder_workers": profile["workers"], + "pipeline_depth": 1, + "deterministic": True, + } + for index, case in enumerate(cases): + case = require_exact_fields( + case, + AGGREGATE_CASE_FIELDS, + f"aggregate case at index {index}", + ) + if case.get("seed") != fixture_cases[index]["seed"]: + raise GateError(f"{case.get('case')} seed mismatch") + logical_files = require_nonnegative_integer( + case.get("logical_files"), f"{case.get('case')} logical_files" + ) + if logical_files <= 0: + raise GateError(f"{case.get('case')} logical_files must be a positive integer") + durations = case.get("sample_durations_ms") + if not isinstance(durations, list) or len(durations) != sample_count: + raise GateError(f"{case.get('case')} sample count mismatch") + expected = summarize(durations) + for field, value in expected.items(): + actual = require_number(case.get(field), f"{case.get('case')} {field}") + if not math.isclose(actual, value, rel_tol=1e-12, abs_tol=1e-9): + raise GateError(f"{case.get('case')} statistic {field} is inconsistent") + logical_bytes = require_nonnegative_integer( + case.get("logical_bytes"), f"{case.get('case')} logical_bytes" + ) + if logical_bytes <= 0: + raise GateError(f"{case.get('case')} logical_bytes must be a positive integer") + if case.get("workers_used") != expected_execution["store_folder_workers"]: + raise GateError(f"{case.get('case')} workers_used mismatch") + first_diagnostic = validate_diagnostic_final_state( + case.get("diagnostic_final_state"), f"aggregate case {case.get('case')!r}" + ) + diagnostics = case.get("diagnostic_samples") + if not isinstance(diagnostics, list) or len(diagnostics) != sample_count: + raise GateError(f"{case.get('case')} diagnostic sample count mismatch") + expected_hard_state = hard_final_state({ + "case": case["case"], + "diagnostic_final_state": first_diagnostic, + }) + for sample_index, diagnostic in enumerate(diagnostics): + diagnostic = validate_diagnostic_final_state( + diagnostic, + f"aggregate case {case.get('case')!r} diagnostic sample {sample_index + 1}", + ) + if hard_final_state({ + "case": case["case"], + "diagnostic_final_state": diagnostic, + }) != expected_hard_state: + raise GateError(f"{case.get('case')} hard diagnostic sample mismatch") + if diagnostics[0] != first_diagnostic: + raise GateError(f"{case.get('case')} first diagnostic sample mismatch") + expected_restored_files = logical_files if case["case"] in { + "restore-large-file", "restore-many-files" + } else 0 + if first_diagnostic["restored_tree"]["count"] != expected_restored_files: + raise GateError(f"{case.get('case')} restored file total mismatch") + validate_operational_distributions( + case.get("operational_samples"), + case.get("operational_counter_distributions"), + sample_count=sample_count, + label=str(case.get("case")), + ) + expected_throughput = ( + logical_bytes + / (1024.0 * 1024.0) + / (expected["median_duration_ms"] / 1000.0) + ) + actual_throughput = require_number( + case.get("throughput_mbps"), + f"{case.get('case')} throughput", + positive=True, + ) + if not math.isclose(actual_throughput, expected_throughput, rel_tol=1e-12, abs_tol=1e-12): + raise GateError(f"{case.get('case')} aggregate throughput is inconsistent") + validate_no_sensitive_evidence(report, "aggregate") + + +def revalidate_raw_command(args: argparse.Namespace) -> int: + raw_paths = sorted(args.raw_dir.glob("sample-*.json")) + if not raw_paths: + raise GateError("revalidation raw directory contains no sample reports") + expected_names = [f"sample-{index:02d}.json" for index in range(1, len(raw_paths) + 1)] + if [path.name for path in raw_paths] != expected_names: + raise GateError("revalidation raw sample ordering is incomplete or non-contiguous") + raw_reports = [load_json_strict(path) for path in raw_paths] + first_data, cases = build_contract_cases( + raw_reports, + workers=args.workers, + compression=args.compression, + ) + report = { + "schema_version": SCHEMA_VERSION, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": REVALIDATION_KIND, + "status": "ok", + "performance_calibration_status": "not_evaluated", + "profile": { + "codec": "aes-gcm", + "compression": args.compression, + "dataset": FIXTURE_ID, + "workers": args.workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "fixture": first_data["fixture"], + "warmup_count": 0, + "sample_count": len(raw_reports), + "sample_order": list(range(1, len(raw_reports) + 1)), + "operation_totals": operation_totals(len(raw_reports)), + "cleanup_totals": cleanup_totals(len(raw_reports)), + "cases": cases, + } + validate_revalidation_report(report) + write_json(args.output, report) + print(args.output) + return 0 + + +def validate_revalidation_report(report: dict[str, Any]) -> None: + """Validate a preserved-raw contract report without claiming calibration.""" + require_exact_fields(report, REVALIDATION_FIELDS, "revalidation report") + if ( + report.get("schema_version") != SCHEMA_VERSION + or report.get("evidence_policy_version") != EVIDENCE_POLICY_VERSION + or report.get("report_kind") != REVALIDATION_KIND + or report.get("status") != "ok" + or report.get("performance_calibration_status") != "not_evaluated" + ): + raise GateError("revalidation schema/policy/kind/status mismatch") + sample_count = require_nonnegative_integer( + report.get("sample_count"), "revalidation sample_count" + ) + if sample_count <= 0 or report.get("warmup_count") != 0: + raise GateError("revalidation requires measured samples and no inferred warmups") + if report.get("sample_order") != list(range(1, sample_count + 1)): + raise GateError("revalidation sample order mismatch") + profile = require_exact_fields(report.get("profile"), PROFILE_FIELDS, "revalidation profile") + if ( + profile.get("codec") != "aes-gcm" + or profile.get("compression") not in {"none", "zstd"} + or profile.get("dataset") != FIXTURE_ID + or profile.get("workers") not in {1, 4} + or profile.get("pipeline_depth") != 1 + or profile.get("deterministic") is not True + ): + raise GateError("revalidation profile mismatch") + validate_fixture(report.get("fixture")) + validate_operation_and_cleanup_totals(report, sample_count=sample_count) + cases = report.get("cases") + if not isinstance(cases, list) or [case.get("case") for case in cases] != EXPECTED_CASES: + raise GateError("revalidation case set/order mismatch") + for case in cases: + require_exact_fields(case, AGGREGATE_CASE_FIELDS, f"revalidation case {case.get('case')!r}") + diagnostics = case.get("diagnostic_samples") + if not isinstance(diagnostics, list) or len(diagnostics) != sample_count: + raise GateError(f"{case.get('case')} revalidation diagnostic sample count mismatch") + first_hard = hard_final_state({ + "case": case.get("case"), + "diagnostic_final_state": case.get("diagnostic_final_state"), + }) + for diagnostic in diagnostics: + if hard_final_state({ + "case": case.get("case"), + "diagnostic_final_state": diagnostic, + }) != first_hard: + raise GateError(f"{case.get('case')} revalidation hard diagnostic mismatch") + validate_operational_distributions( + case.get("operational_samples"), + case.get("operational_counter_distributions"), + sample_count=sample_count, + label=f"{case.get('case')} revalidation", + ) + validate_no_sensitive_evidence(report, "revalidation report") + + +def threshold_policy(path: pathlib.Path, mode: str) -> tuple[Decimal, dict[str, Decimal], bool]: + lines = path.read_text(encoding="utf-8").splitlines() + if mode == "uncompressed": + default_key = "duration_regression_pct" + hard_fail = True + else: + default_key = "duration_regression_warning_pct" + hard_fail = False + + default: Decimal | None = None + overrides: dict[str, Decimal] = {} + section = "" + current_mode = "" + current_case = "" + in_defaults = False + in_overrides = False + for raw in lines: + text = raw.split("#", 1)[0].rstrip() + if not text.strip(): + continue + indent = len(text) - len(text.lstrip()) + stripped = text.strip() + if indent == 0 and stripped.endswith(":"): + section = stripped[:-1] + in_defaults = section == "defaults" + in_overrides = section == "per_case_overrides" + current_mode = "" + current_case = "" + continue + if (in_defaults or in_overrides) and indent == 2 and stripped.endswith(":"): + current_mode = stripped[:-1] + current_case = "" + continue + if in_overrides and current_mode == mode and indent == 4 and stripped.endswith(":"): + current_case = stripped[:-1] + continue + match = re.fullmatch(r"([a-z_]+):\s*([0-9]+(?:\.[0-9]+)?)", stripped) + if not match or match.group(1) != default_key or current_mode != mode: + continue + value = Decimal(match.group(2)) + if in_defaults and indent == 4: + default = value + elif in_overrides and indent == 6 and current_case: + overrides[current_case] = value + if default is None: + raise GateError(f"cannot locate {mode} default duration threshold in {path}") + return default, overrides, hard_fail + + +def compare_command(args: argparse.Namespace) -> int: + candidate = load_json_strict(args.candidate) + baseline = load_json_strict(args.baseline) + validate_aggregate(candidate, require_gate_count=True) + validate_aggregate(baseline, require_gate_count=True) + if args.manifest: + manifest = validate_manifest(args.manifest) + baseline_hash = sha256_file(args.baseline) + profile_key = f"{baseline['profile']['compression']}-w{baseline['profile']['workers']}" + if manifest["artifacts"][profile_key].get("sha256") != baseline_hash: + raise GateError("selected baseline is not owned by the authoritative manifest") + if manifest["thresholds"]["sha256"] != sha256_file(args.thresholds): + raise GateError("threshold file hash does not match the authoritative manifest") + + if candidate["profile"] != baseline["profile"]: + raise GateError("candidate and baseline profile metadata differ") + expected_mode = "uncompressed" if baseline["profile"]["compression"] == "none" else "compressed" + if args.mode != expected_mode: + raise GateError("comparison mode does not match compression profile") + if candidate["fixture"] != baseline["fixture"]: + raise GateError("candidate and baseline fixture metadata differ") + for field in HARD_ENV_FIELDS: + if candidate["provenance"][field] != baseline["provenance"][field]: + raise GateError(f"environment provenance mismatch for {field}") + warnings = [] + if candidate["provenance"].get("runner_image") != baseline["provenance"].get("runner_image"): + warnings.append("resolved runner image differs from baseline") + + default_threshold, overrides, hard_fail_policy = threshold_policy(args.thresholds, args.mode) + outcomes = [] + has_instability = False + has_hard_regression = False + for baseline_case, candidate_case in zip(baseline["cases"], candidate["cases"]): + case_name = baseline_case["case"] + if hard_aggregate_case_contract(candidate_case) != hard_aggregate_case_contract(baseline_case): + raise GateError(f"hard case evidence differs for {case_name}") + threshold = overrides.get(case_name, default_threshold) + variability_limit = min(Decimal("2"), threshold / Decimal("2")) + base_median = Decimal(str(baseline_case["median_duration_ms"])) + candidate_median = Decimal(str(candidate_case["median_duration_ms"])) + baseline_variability = Decimal(str(baseline_case["mad_ratio_pct"])) + candidate_variability = Decimal(str(candidate_case["mad_ratio_pct"])) + if base_median < Decimal("5000"): + raise GateError(f"baseline case {case_name} is shorter than 5000 ms") + classification = "pass" + if baseline_variability > variability_limit or candidate_variability > variability_limit: + classification = "BENCHMARK_UNSTABLE" + has_instability = True + delta = (candidate_median - base_median) * Decimal("100") / base_median + if classification == "pass" and delta > threshold: + classification = "PERFORMANCE_REGRESSION" if hard_fail_policy else "PERFORMANCE_WARNING" + has_hard_regression = has_hard_regression or hard_fail_policy + outcomes.append( + { + "case": case_name, + "classification": classification, + "baseline_median_ms": float(base_median), + "candidate_median_ms": float(candidate_median), + "delta_pct": float(delta), + "threshold_pct": float(threshold), + "variability_limit_pct": float(variability_limit), + "baseline_operational_counters": baseline_case["operational_counter_distributions"], + "candidate_operational_counters": candidate_case["operational_counter_distributions"], + } + ) + + passed = not has_instability and not has_hard_regression + report = { + "schema_version": SCHEMA_VERSION, + "report_kind": "benchmark_gate_comparison", + "status": "ok" if passed else "failed", + "manifest_sha256": sha256_file(args.manifest) if args.manifest else None, + "warnings": warnings, + "outcomes": outcomes, + } + write_json(args.output, report) + return 0 if passed else 1 + + +def calibration_command(args: argparse.Namespace) -> int: + reports: dict[tuple[str, int, int], dict[str, Any]] = {} + for item in args.aggregate: + if "=" not in item: + raise GateError("--aggregate values must use compression-wN-rN=path") + label, raw_path = item.split("=", 1) + match = re.fullmatch(r"(none|zstd)-w(1|4)-r(1|2)", label) + if not match: + raise GateError(f"invalid calibration label {label!r}") + key = (match.group(1), int(match.group(2)), int(match.group(3))) + if key in reports: + raise GateError(f"duplicate calibration profile {label!r}") + report = load_json_strict(pathlib.Path(raw_path)) + validate_aggregate(report, require_gate_count=False) + if report["sample_count"] != 10 or report["warmup_count"] != 1: + raise GateError(f"calibration profile {label!r} requires one warmup and ten samples") + expected_profile = { + "codec": "aes-gcm", + "compression": key[0], + "dataset": FIXTURE_ID, + "workers": key[1], + "pipeline_depth": 1, + "deterministic": True, + } + if report["profile"] != expected_profile: + raise GateError(f"calibration profile metadata mismatch for {label!r}") + reports[key] = report + expected_keys = { + (compression, workers, replicate) + for compression in ("none", "zstd") + for workers in (1, 4) + for replicate in (1, 2) + } + if set(reports) != expected_keys: + raise GateError("calibration requires exactly two replicas of all four profiles") + + failures = [] + profile_results = [] + for compression in ("none", "zstd"): + mode = "uncompressed" if compression == "none" else "compressed" + default_threshold, overrides, _ = threshold_policy(args.thresholds, mode) + for workers in (1, 4): + first = reports[(compression, workers, 1)] + second = reports[(compression, workers, 2)] + for field in CALIBRATION_IDENTITY_FIELDS: + if first["provenance"][field] != second["provenance"][field]: + failures.append( + f"{compression}-w{workers}: replica environment mismatch for {field}" + ) + for report in (first, second): + if report["command_p95_ms"] > 120_000: + failures.append(f"{compression}-w{workers}: command p95 exceeds 120 seconds") + for first_case, second_case in zip(first["cases"], second["cases"]): + case_name = first_case["case"] + threshold = overrides.get(case_name, default_threshold) + variability_limit = float(min(Decimal("2"), threshold / Decimal("2"))) + if hard_aggregate_case_contract(first_case) != hard_aggregate_case_contract(second_case): + failures.append(f"{compression}-w{workers}/{case_name}: replica hard evidence mismatch") + replica_medians = [] + for replicate, case in ((1, first_case), (2, second_case)): + median = float(case["median_duration_ms"]) + replica_medians.append(median) + if median < 5000: + failures.append( + f"{compression}-w{workers}-r{replicate}/{case_name}: median below 5000 ms" + ) + if float(case["mad_ratio_pct"]) > variability_limit: + failures.append( + f"{compression}-w{workers}-r{replicate}/{case_name}: ten-sample MAD ratio exceeds {variability_limit}%" + ) + durations = case["sample_durations_ms"] + for subset_name, subset in ( + ("odd", durations[0::2]), + ("even", durations[1::2]), + ): + subset_stats = summarize(subset) + if subset_stats["mad_ratio_pct"] > variability_limit: + failures.append( + f"{compression}-w{workers}-r{replicate}/{case_name}: {subset_name} five-sample MAD ratio exceeds {variability_limit}%" + ) + replica_delta = ( + abs(replica_medians[1] - replica_medians[0]) + / min(replica_medians) + * 100.0 + ) + if replica_delta > 5.0: + failures.append( + f"{compression}-w{workers}/{case_name}: replica medians differ by {replica_delta:.3f}%" + ) + profile_results.append( + { + "profile": f"{compression}-w{workers}", + "case": case_name, + "replica_medians_ms": replica_medians, + "replica_delta_pct": replica_delta, + "variability_limit_pct": variability_limit, + } + ) + result = { + "schema_version": SCHEMA_VERSION, + "report_kind": "benchmark_gate_calibration", + "status": "ok" if not failures else "failed", + "failures": failures, + "profiles": profile_results, + } + write_json(args.output, result) + return 0 if not failures else 1 + + +def manifest_command(args: argparse.Namespace) -> int: + repository_root = pathlib.Path.cwd().resolve() + + def repository_relative(path: pathlib.Path) -> str: + try: + return path.resolve().relative_to(repository_root).as_posix() + except ValueError as exc: + raise GateError(f"manifest path must be repository-relative: {path}") from exc + + artifacts: dict[str, Any] = {} + source_commits: set[str] = set() + for item in args.baseline: + if "=" not in item: + raise GateError("--baseline values must use profile=path") + profile, raw_path = item.split("=", 1) + if profile not in MANIFEST_PROFILES: + raise GateError(f"invalid manifest profile {profile!r}") + path = pathlib.Path(raw_path) + report = load_json_strict(path) + validate_aggregate(report, require_gate_count=True) + if profile in artifacts: + raise GateError(f"duplicate manifest profile {profile!r}") + expected_compression, expected_workers = MANIFEST_PROFILES[profile] + if ( + report["profile"]["compression"] != expected_compression + or report["profile"]["workers"] != expected_workers + ): + raise GateError(f"manifest profile {profile!r} does not match aggregate metadata") + source_commits.add(report["provenance"]["source_commit"]) + artifacts[profile] = { + "path": repository_relative(path), + "sha256": sha256_file(path), + "source_commit": report["provenance"]["source_commit"], + } + if set(artifacts) != set(MANIFEST_PROFILES): + raise GateError("authoritative manifest requires exactly four profiles") + if len(source_commits) != 1: + raise GateError("authoritative baselines must share one governed source commit") + manifest = { + "schema_version": SCHEMA_VERSION, + "manifest_kind": MANIFEST_KIND, + "generated_at_utc": utc_now(), + "thresholds": { + "path": repository_relative(args.thresholds), + "sha256": sha256_file(args.thresholds), + }, + "artifacts": dict(sorted(artifacts.items())), + } + write_json(args.output, manifest) + return 0 + + +def validate_manifest_command(args: argparse.Namespace) -> int: + validate_manifest(args.manifest) + print("benchmark gate manifest is valid") + return 0 + + +def validate_manifest(path: pathlib.Path) -> dict[str, Any]: + manifest = load_json_strict(path) + if ( + manifest.get("schema_version") != SCHEMA_VERSION + or manifest.get("manifest_kind") != MANIFEST_KIND + ): + raise GateError("manifest schema/kind mismatch") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, dict) or set(artifacts) != set(MANIFEST_PROFILES): + raise GateError("manifest must contain exactly four artifacts") + source_commits: set[str] = set() + entries = {**artifacts, "_thresholds": manifest.get("thresholds")} + for label, entry in entries.items(): + if not isinstance(entry, dict): + raise GateError(f"manifest entry {label!r} must be an object") + path = pathlib.Path(entry.get("path", "")) + if path.is_absolute() or ".." in path.parts: + raise GateError(f"manifest path for {label} must be repository-relative") + if not path.is_file() or sha256_file(path) != entry.get("sha256"): + raise GateError(f"manifest hash mismatch for {label}") + if label != "_thresholds": + report = load_json_strict(path) + validate_aggregate(report, require_gate_count=True) + expected_compression, expected_workers = MANIFEST_PROFILES[label] + if ( + report["profile"]["compression"] != expected_compression + or report["profile"]["workers"] != expected_workers + or entry.get("source_commit") != report["provenance"]["source_commit"] + ): + raise GateError(f"manifest profile metadata mismatch for {label}") + source_commits.add(report["provenance"]["source_commit"]) + if len(source_commits) != 1: + raise GateError("manifest artifacts do not share one governed source commit") + return manifest + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + sample = subparsers.add_parser("sample", help="capture independent benchmark samples") + sample.add_argument("--binary", type=pathlib.Path, required=True) + sample.add_argument("--output-dir", type=pathlib.Path, required=True) + sample.add_argument("--compression", choices=("none", "zstd"), required=True) + sample.add_argument("--workers", type=int, choices=(1, 4), required=True) + sample.add_argument("--dataset", default=FIXTURE_ID) + sample.add_argument("--warmups", type=int, default=1) + sample.add_argument("--samples", type=int, default=5) + sample.add_argument("--minimum-free-disk-bytes", type=int, default=10 * 1024**3) + sample.add_argument("--source-commit") + sample.add_argument("--source-tag") + sample.add_argument("--go-version") + sample.add_argument("--postgres-version", required=True) + sample.add_argument("--database-image-digest", required=True) + sample.set_defaults(handler=sample_command) + + integrity = subparsers.add_parser( + "integrity", + help="capture two candidate-only v2 samples with hard functional evidence", + ) + integrity.add_argument("--binary", type=pathlib.Path, required=True) + integrity.add_argument("--output-dir", type=pathlib.Path, required=True) + integrity.add_argument("--compression", choices=("none", "zstd"), required=True) + integrity.add_argument("--workers", type=int, choices=(1, 4), required=True) + integrity.add_argument("--dataset", choices=tuple(INTEGRITY_FIXTURES), required=True) + integrity.add_argument( + "--command-timeout-seconds", + type=int, + default=INTEGRITY_COMMAND_TIMEOUT_SECONDS, + ) + integrity.add_argument("--source-commit") + integrity.add_argument("--source-tag") + integrity.add_argument("--go-version") + integrity.add_argument("--postgres-version", required=True) + integrity.add_argument("--database-image-digest", required=True) + integrity.set_defaults(handler=integrity_command) + + revalidate = subparsers.add_parser( + "revalidate-raw", + help="validate preserved raw diagnostic reports without claiming calibration acceptance", + ) + revalidate.add_argument("--raw-dir", type=pathlib.Path, required=True) + revalidate.add_argument("--compression", choices=("none", "zstd"), required=True) + revalidate.add_argument("--workers", type=int, choices=(1, 4), required=True) + revalidate.add_argument("--output", type=pathlib.Path, required=True) + revalidate.set_defaults(handler=revalidate_raw_command) + + compare = subparsers.add_parser("compare", help="compare aggregate evidence") + compare.add_argument("--candidate", type=pathlib.Path, required=True) + compare.add_argument("--baseline", type=pathlib.Path, required=True) + compare.add_argument("--thresholds", type=pathlib.Path, required=True) + compare.add_argument("--mode", choices=("uncompressed", "compressed"), required=True) + compare.add_argument("--manifest", type=pathlib.Path) + compare.add_argument("--output", type=pathlib.Path, required=True) + compare.set_defaults(handler=compare_command) + + calibrate = subparsers.add_parser("calibrate", help="evaluate fixed calibration evidence") + calibrate.add_argument("--aggregate", action="append", default=[], required=True) + calibrate.add_argument("--thresholds", type=pathlib.Path, required=True) + calibrate.add_argument("--output", type=pathlib.Path, required=True) + calibrate.set_defaults(handler=calibration_command) + + manifest = subparsers.add_parser("manifest", help="generate an authoritative manifest") + manifest.add_argument("--baseline", action="append", default=[], required=True) + manifest.add_argument("--thresholds", type=pathlib.Path, required=True) + manifest.add_argument("--output", type=pathlib.Path, required=True) + manifest.set_defaults(handler=manifest_command) + + validate_manifest_parser = subparsers.add_parser( + "validate-manifest", help="validate manifest hashes and aggregates" + ) + validate_manifest_parser.add_argument("--manifest", type=pathlib.Path, required=True) + validate_manifest_parser.set_defaults(handler=validate_manifest_command) + return parser + + +def main() -> int: + args = build_parser().parse_args() + try: + return args.handler(args) + except (GateError, OSError, subprocess.SubprocessError) as exc: + print(f"benchmark gate error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/paired_benchmark_gate.py b/scripts/paired_benchmark_gate.py new file mode 100644 index 00000000..b35ea648 --- /dev/null +++ b/scripts/paired_benchmark_gate.py @@ -0,0 +1,3287 @@ +#!/usr/bin/env python3 +""" +Run and validate Coldkeep's same-job reference/candidate benchmark gate. + +Raw benchmark and diagnostic-final-state payloads remain schema version 2. This +module adds the independent ``benchmark_paired_comparison`` schema version 1. +It intentionally contains no production threshold values and no default +reference: both are governed inputs that must be added in later stages. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import pathlib +import platform +import re +import shutil +import signal +import statistics +import subprocess +import sys +import tempfile +import time +from decimal import Decimal +from typing import Any, NoReturn + +import benchmark_gate as raw_gate + + +REPORT_KIND = "benchmark_paired_comparison" +DECISION_KIND = "benchmark_paired_decision" +REFERENCE_MANIFEST_KIND = "benchmark_paired_reference_manifest" +THRESHOLD_POLICY_KIND = "benchmark_paired_threshold_policy" +SCHEMA_VERSION = 1 +EVIDENCE_POLICY_VERSION = 2 +CONTRACT_VERSION = "coldkeep-paired-v1" +RAW_SCHEMA_VERSION = 2 +DIAGNOSTIC_SCHEMA_VERSION = 2 +DECISION_MODES = ("diagnostic", "production") +DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS = 35 * 60 * 1000 +DIAGNOSTIC_MAX_PROFILE_ELAPSED_SECONDS = DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS / 1000 +PROCESS_TERMINATION_GRACE_SECONDS = 10 +GOVERNED_MANIFEST_RELATIVE = pathlib.PurePosixPath( + "benchmarks/paired/reference-v1.13.json" +) +GOVERNED_THRESHOLD_RELATIVE = pathlib.PurePosixPath( + "benchmarks/paired/threshold-policy-v1.13.json" +) +# A later, separately authorized governance commit must enable production only +# after adding the fixed manifest/policy and trusted-base workflow integration. +PRODUCTION_SAMPLING_AUTHORIZED = False + +SENSITIVE_CAPTURE_PATTERNS = ( + re.compile(r"(?i)\b(?:postgres(?:ql)?|mysql|mariadb)://"), + re.compile( + r"(?i)\b(?:password|passwd|credential|encryption_key|dsn|dbname|" + r"database_name|db_name|username|user_name)\s*[:=]" + ), + re.compile(r"(?i)\b(?:DB_PASSWORD|COLDKEEP_KEY|DATABASE_URL|PGPASSWORD)\b"), + re.compile(r"(?i)(?:^|[\s'\"=])/(?:home|tmp|var|workspaces)/[^\s'\"]+"), + re.compile(r"(?i)(?:^|[\s'\"=])[A-Z]:[\\/][^\s'\"]+"), + re.compile(r"(?i)coldkeep[_-]bench(?:mark)?[_-]"), +) + +ORDERED_CASES = tuple(raw_gate.EXPECTED_CASES) +PERFORMANCE_CASES = ( + "store-large-file", + "store-many-small-files", + "restore-many-files", + "snapshot-creation", + "gc-after-churn", + "stats-inspect", + "verify-system-deep", +) +WARMUP_ORDER = ("candidate", "reference") +FIVE_PAIR_ORDER: tuple[tuple[str, str], ...] = ( + ("reference", "candidate"), + ("candidate", "reference"), + ("candidate", "reference"), + ("reference", "candidate"), + ("reference", "candidate"), +) +TEN_PAIR_ORDER: tuple[tuple[str, str], ...] = FIVE_PAIR_ORDER + tuple( + (pair_order[1], pair_order[0]) for pair_order in FIVE_PAIR_ORDER +) +PROFILE_MATRIX = { + "none-w1": ("none", 1, "ci-paired-w1-v2"), + "none-w4": ("none", 4, "ci-paired-w4-v2"), + "zstd-w1": ("zstd", 1, "ci-paired-w1-v2"), + "zstd-w4": ("zstd", 4, "ci-paired-w4-v2"), +} +FIXTURES = { + "ci-paired-w1-v1": { + "id": "ci-paired-w1-v1", + "seed": 1701, + "large_file_size_bytes": 96 * 1024 * 1024, + "many_small_file_count": 600, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 400, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 1, + }, + "ci-paired-w4-v1": { + "id": "ci-paired-w4-v1", + "seed": 1701, + "large_file_size_bytes": 128 * 1024 * 1024, + "many_small_file_count": 1200, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 800, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 4, + }, + "ci-paired-w1-v2": { + "id": "ci-paired-w1-v2", + "seed": 1701, + "large_file_size_bytes": 64 * 1024 * 1024, + "many_small_file_count": 400, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 400, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 1, + }, + "ci-paired-w4-v2": { + "id": "ci-paired-w4-v2", + "seed": 1701, + "large_file_size_bytes": 64 * 1024 * 1024, + "many_small_file_count": 400, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 800, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 4, + }, +} + +CLASSIFICATIONS = { + "CONTRACT_INVALID", + "PAIR_INVENTORY_INVALID", + "REFERENCE_GOVERNANCE_INVALID", + "BINARY_IDENTITY_INVALID", + "EXECUTION_CONTRACT_MISMATCH", + "REFERENCE_FUNCTIONAL_FAILURE", + "CANDIDATE_FUNCTIONAL_FAILURE", + "CORRECTNESS_REGRESSION", + "EVIDENCE_INTEGRITY_FAILURE", + "BENCHMARK_ENVIRONMENT_UNSTABLE", + "PERFORMANCE_REGRESSION", + "CANDIDATE_TIMEOUT_INCONCLUSIVE", + "CI_INFRASTRUCTURE_TIMEOUT", + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + "DIAGNOSTIC_QUALIFIED", + "DIAGNOSTIC_REJECTED", + "PASS", +} +SUCCESS_CLASSIFICATIONS = {"PASS", "DIAGNOSTIC_QUALIFIED"} +DECISION_PRECEDENCE = ( + "CONTRACT_INVALID", + "PAIR_INVENTORY_INVALID", + "REFERENCE_GOVERNANCE_INVALID", + "BINARY_IDENTITY_INVALID", + "EXECUTION_CONTRACT_MISMATCH", + "REFERENCE_FUNCTIONAL_FAILURE", + "CANDIDATE_FUNCTIONAL_FAILURE", + "CANDIDATE_TIMEOUT_INCONCLUSIVE", + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + "CI_INFRASTRUCTURE_TIMEOUT", + "CORRECTNESS_REGRESSION", + "EVIDENCE_INTEGRITY_FAILURE", + "BENCHMARK_ENVIRONMENT_UNSTABLE", + "PERFORMANCE_REGRESSION", + "DIAGNOSTIC_REJECTED", + "DIAGNOSTIC_QUALIFIED", + "PASS", +) + + +class PairedGateError(raw_gate.GateError): + """A fail-closed paired-gate error with a stable classification.""" + + def __init__(self, classification: str, message: str): + """Initialize an error with a validated stable classification.""" + if classification not in CLASSIFICATIONS - SUCCESS_CLASSIFICATIONS: + raise ValueError(f"invalid paired classification {classification!r}") + super().__init__(message) + self.classification = classification + + +def fail(classification: str, message: str) -> NoReturn: + raise PairedGateError(classification, message) + + +def _git_executable() -> str: + executable = shutil.which("git") + if executable is None or not pathlib.Path(executable).is_absolute(): + fail("REFERENCE_GOVERNANCE_INVALID", "cannot resolve an absolute git executable") + return executable + + +def authority_contract(mode: str) -> dict[str, Any]: + if mode == "diagnostic": + return { + "decision_scope": "diagnostic_qualification", + "authority": "diagnostic_only", + "production_authority": False, + } + if mode == "production": + return { + "decision_scope": "production_regression", + "authority": "governed_production", + "production_authority": True, + } + fail("CONTRACT_INVALID", f"unknown decision mode {mode!r}") + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, nested in pairs: + if key in value: + raise raw_gate.GateError(f"duplicate JSON key {key!r}") + value[key] = nested + return value + + +def load_json_strict(path: pathlib.Path) -> dict[str, Any]: + if path.is_symlink() or not path.is_file(): + raise raw_gate.GateError(f"JSON input must be a regular non-symlink file: {path}") + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise raw_gate.GateError(f"read JSON input {path}: {exc}") from exc + decoder = json.JSONDecoder( + object_pairs_hook=_reject_duplicate_json_keys, + parse_constant=lambda value: (_ for _ in ()).throw( + raw_gate.GateError(f"non-finite JSON value {value!r}") + ), + ) + try: + value, end = decoder.raw_decode(text) + except (json.JSONDecodeError, raw_gate.GateError) as exc: + raise raw_gate.GateError(f"malformed JSON in {path}: {exc}") from exc + if text[end:].strip(): + raise raw_gate.GateError(f"trailing JSON or content in {path}") + if not isinstance(value, dict): + raise raw_gate.GateError(f"top-level JSON value in {path} must be an object") + return value + + +def require_relative_artifact_path(value: Any, label: str) -> pathlib.PurePosixPath: + if not isinstance(value, str) or not value or "\\" in value: + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} must be a normalized relative path") + relative = pathlib.PurePosixPath(value) + if ( + relative.is_absolute() + or value != relative.as_posix() + or any(part in {"", ".", ".."} for part in relative.parts) + ): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} must be a contained relative path") + return relative + + +def _contained_regular_file( + directory: pathlib.Path, relative: pathlib.PurePosixPath, label: str +) -> pathlib.Path: + if directory.is_symlink() or not directory.is_dir(): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} directory is not a regular directory") + root = directory.resolve() + path = directory.joinpath(*relative.parts) + if path.is_symlink() or not path.is_file(): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} is missing or is a symlink") + try: + path.resolve().relative_to(root) + except ValueError: + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} escapes the artifact directory") + cursor = path.parent + while cursor != directory: + if cursor.is_symlink(): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} traverses a symlink directory") + cursor = cursor.parent + return path + + +def _artifact_files(directory: pathlib.Path) -> set[str]: + if directory.is_symlink() or not directory.is_dir(): + fail("EVIDENCE_INTEGRITY_FAILURE", "artifact root must be a non-symlink directory") + files: set[str] = set() + for path in directory.rglob("*"): + if path.is_symlink(): + fail("EVIDENCE_INTEGRITY_FAILURE", "artifact contains a symlink") + if path.is_file(): + files.add(path.relative_to(directory).as_posix()) + return files + + +def _capture_text_is_sensitive(value: str) -> bool: + try: + raw_gate.validate_no_sensitive_evidence({"capture": value}, "captured output") + except raw_gate.GateError: + return True + return any(pattern.search(value) for pattern in SENSITIVE_CAPTURE_PATTERNS) + + +def _sanitize_failure_captures(directory: pathlib.Path) -> None: + raw_root = directory / "raw" + if not raw_root.is_dir(): + return + for path in raw_root.rglob("*"): + if path.is_symlink() or not path.is_file(): + continue + try: + value = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + path.write_text("[captured output omitted: invalid text]\n", encoding="utf-8") + continue + if "\x00" in value or _capture_text_is_sensitive(value): + path.write_text("[captured output omitted: sensitive content]\n", encoding="utf-8") + + +def _create_output_directory(path: pathlib.Path) -> None: + if path.exists() or path.is_symlink(): + fail("EVIDENCE_INTEGRITY_FAILURE", "output directory already exists") + try: + path.mkdir(parents=True, exist_ok=False) + except OSError as exc: + fail("EVIDENCE_INTEGRITY_FAILURE", f"create output directory: {exc}") + if path.is_symlink() or not path.is_dir(): + fail("EVIDENCE_INTEGRITY_FAILURE", "output directory is not a regular directory") + + +def _repository_root() -> pathlib.Path: + completed = subprocess.run( + [_git_executable(), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + fail("REFERENCE_GOVERNANCE_INVALID", "cannot resolve repository root") + return pathlib.Path(completed.stdout.strip()).resolve() + + +def _governed_repository_file( + repository: pathlib.Path, relative: pathlib.PurePosixPath, label: str +) -> pathlib.Path: + root = repository.resolve() + path = repository.joinpath(*relative.parts) + if path.is_symlink() or not path.is_file(): + fail("REFERENCE_GOVERNANCE_INVALID", f"{label} is absent or is a symlink") + try: + path.resolve().relative_to(root) + except ValueError: + fail("REFERENCE_GOVERNANCE_INVALID", f"{label} escapes the repository") + cursor = path.parent + while cursor != repository: + if cursor.is_symlink(): + fail("REFERENCE_GOVERNANCE_INVALID", f"{label} traverses a symlink directory") + cursor = cursor.parent + return path + + +def require_sha(value: Any, label: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value): + fail("REFERENCE_GOVERNANCE_INVALID", f"{label} must be a lowercase 40-character SHA") + return value + + +def validate_repository_id(value: Any) -> str: + if value == "local": + return value + if not isinstance(value, str) or not re.fullmatch( + r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", value + ): + fail("EXECUTION_CONTRACT_MISMATCH", "repository identity is not sanitized") + return value + + +def profile_artifact_name( + *, candidate_sha: str, reference_sha: str, compression: str, workers: int, attempt: int +) -> str: + require_sha(candidate_sha, "candidate SHA") + require_sha(reference_sha, "reference SHA") + if compression not in {"none", "zstd"} or workers not in {1, 4}: + fail("EXECUTION_CONTRACT_MISMATCH", "artifact profile identity is invalid") + if isinstance(attempt, bool) or not isinstance(attempt, int) or attempt <= 0: + fail("CONTRACT_INVALID", "artifact attempt must be a positive integer") + return ( + f"benchmark-paired-{candidate_sha[:12]}-against-{reference_sha[:12]}-" + f"{compression}-w{workers}-a{attempt}" + ) + + +def decision_artifact_name(*, candidate_sha: str, reference_sha: str, attempt: int) -> str: + require_sha(candidate_sha, "candidate SHA") + require_sha(reference_sha, "reference SHA") + if isinstance(attempt, bool) or not isinstance(attempt, int) or attempt <= 0: + fail("CONTRACT_INVALID", "artifact attempt must be a positive integer") + return ( + f"benchmark-paired-{candidate_sha[:12]}-against-{reference_sha[:12]}-" + f"decision-a{attempt}" + ) + + +def measured_order(pair_count: int) -> tuple[tuple[str, str], ...]: + if pair_count == 5: + return FIVE_PAIR_ORDER + if pair_count == 10: + return TEN_PAIR_ORDER + fail("PAIR_INVENTORY_INVALID", "paired sampling requires exactly 5 or 10 pairs") + raise AssertionError("unreachable") + + +def fixture_contract(dataset: str, workers: int) -> dict[str, Any]: + expected = FIXTURES.get(dataset) + if expected is None: + fail("EXECUTION_CONTRACT_MISMATCH", f"unsupported paired fixture {dataset!r}") + if expected["workers"] != workers: + fail( + "EXECUTION_CONTRACT_MISMATCH", + f"fixture {dataset!r} requires workers={expected['workers']}", + ) + return {key: value for key, value in expected.items() if key != "workers"} + + +def validate_fixture(value: Any, *, dataset: str, workers: int) -> dict[str, Any]: + expected = fixture_contract(dataset, workers) + try: + fixture = raw_gate.require_exact_fields( + value, set(expected) | {"ordered_cases"}, "paired fixture" + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + for field, expected_value in expected.items(): + if fixture.get(field) != expected_value: + fail( + "EXECUTION_CONTRACT_MISMATCH", + f"paired fixture field {field!r} does not match {dataset!r}", + ) + ordered = fixture["ordered_cases"] + if not isinstance(ordered, list) or len(ordered) != len(ORDERED_CASES): + fail("CONTRACT_INVALID", "paired fixture ordered case count mismatch") + for index, expected_name in enumerate(ORDERED_CASES): + try: + descriptor = raw_gate.require_exact_fields( + ordered[index], {"name", "seed"}, f"paired fixture case {index + 1}" + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if descriptor["name"] != expected_name or descriptor["seed"] != 1712 + 10 * index: + fail("EXECUTION_CONTRACT_MISMATCH", f"paired fixture case {index + 1} mismatch") + return fixture + + +def validate_raw_report( + envelope: Any, *, dataset: str, workers: int, compression: str +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Validate raw schema v2 without changing the legacy fixture authority.""" + try: + envelope = raw_gate.require_exact_fields( + envelope, raw_gate.RAW_ENVELOPE_FIELDS, "raw report envelope" + ) + if envelope["status"] != "ok" or envelope["command"] != "benchmark": + fail("CONTRACT_INVALID", "raw report must be a successful benchmark envelope") + data = raw_gate.require_exact_fields( + envelope["data"], raw_gate.RAW_DATA_FIELDS, "raw report data" + ) + if data["schema_version"] != RAW_SCHEMA_VERSION: + fail("CONTRACT_INVALID", f"raw report schema must be {RAW_SCHEMA_VERSION}") + if data["dataset"] != dataset or data["repeat"] != 1: + fail("EXECUTION_CONTRACT_MISMATCH", "raw dataset or repeat count mismatch") + execution = raw_gate.validate_execution( + data["execution"], workers=workers, label="raw report execution" + ) + validate_fixture(data["fixture"], dataset=dataset, workers=workers) + rows = data["rows"] + if not isinstance(rows, list) or len(rows) != len(ORDERED_CASES): + fail("CONTRACT_INVALID", "raw report row count mismatch") + names = [row.get("case") for row in rows if isinstance(row, dict)] + if names != list(ORDERED_CASES) or len(set(names)) != len(names): + fail("CONTRACT_INVALID", "raw report case set/order mismatch") + for row in rows: + row = raw_gate.require_exact_fields( + row, raw_gate.RAW_ROW_FIELDS, f"raw row {row.get('case')!r}" + ) + duration = raw_gate.require_number( + row["duration_ms"], f"{row['case']} duration", positive=True + ) + throughput = raw_gate.require_number( + row["throughput_mbps"], f"{row['case']} throughput", positive=True + ) + row_execution = raw_gate.validate_execution( + row["execution"], workers=workers, label=f"{row['case']} execution" + ) + if row_execution != execution: + fail("EXECUTION_CONTRACT_MISMATCH", f"{row['case']} execution mismatch") + raw_gate.validate_operational_counters(row, workers=workers) + expected_throughput = ( + row["execution_stats"]["total_bytes"] + / (1024.0 * 1024.0) + / (duration / 1000.0) + ) + if not math.isclose(throughput, expected_throughput, rel_tol=1e-12, abs_tol=1e-12): + fail("CONTRACT_INVALID", f"{row['case']} throughput is inconsistent") + raw_gate.hard_final_state(row) + raw_gate.validate_top_execution_stats(data["execution_stats"], rows, workers=workers) + if compression not in {"none", "zstd"}: + fail("EXECUTION_CONTRACT_MISMATCH", f"unsupported compression {compression!r}") + raw_gate.validate_no_sensitive_evidence(envelope, "paired raw report") + return data, rows + except PairedGateError: + raise + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + raise AssertionError("unreachable") + + +def validate_threshold_policy(value: Any) -> dict[str, float]: + try: + value = raw_gate.require_exact_fields( + value, + {"schema_version", "report_kind", "contract_version", "policy_id", "cases"}, + "paired threshold policy", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if ( + value["schema_version"] != 1 + or value["report_kind"] != THRESHOLD_POLICY_KIND + or value["contract_version"] != CONTRACT_VERSION + or not isinstance(value["policy_id"], str) + or not value["policy_id"] + ): + fail("CONTRACT_INVALID", "paired threshold policy identity mismatch") + cases = value["cases"] + if not isinstance(cases, dict) or set(cases) != set(PERFORMANCE_CASES): + fail("CONTRACT_INVALID", "paired threshold case set mismatch") + normalized: dict[str, float] = {} + for case_name in PERFORMANCE_CASES: + try: + threshold = raw_gate.require_number( + cases[case_name], f"threshold for {case_name}", positive=True + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if threshold > 10.0: + fail("CONTRACT_INVALID", f"threshold for {case_name} exceeds 10%") + normalized[case_name] = threshold + return normalized + + +def validate_reference_manifest(value: Any) -> dict[str, Any]: + expected_fields = { + "schema_version", + "report_kind", + "release_train", + "reference_sha", + "approval", + "contract_version", + "raw_schema_version", + "diagnostic_schema_version", + "fixtures", + "ordered_cases", + "performance_cases", + "execution_order", + "pair_count", + "threshold_policy_id", + "threshold_policy_sha256", + } + try: + value = raw_gate.require_exact_fields(value, expected_fields, "reference manifest") + approval = raw_gate.require_exact_fields( + value["approval"], {"kind", "value"}, "reference manifest approval" + ) + except raw_gate.GateError as exc: + fail("REFERENCE_GOVERNANCE_INVALID", str(exc)) + if value["schema_version"] != 1 or value["report_kind"] != REFERENCE_MANIFEST_KIND: + fail("REFERENCE_GOVERNANCE_INVALID", "reference manifest identity mismatch") + if value["release_train"] != "v1.13" or value["contract_version"] != CONTRACT_VERSION: + fail("REFERENCE_GOVERNANCE_INVALID", "reference manifest contract mismatch") + require_sha(value["reference_sha"], "reference manifest SHA") + if approval["kind"] not in {"trusted_tag", "reviewed_record"} or not approval["value"]: + fail("REFERENCE_GOVERNANCE_INVALID", "reference approval record is invalid") + if ( + value["raw_schema_version"] != RAW_SCHEMA_VERSION + or value["diagnostic_schema_version"] != DIAGNOSTIC_SCHEMA_VERSION + ): + fail("REFERENCE_GOVERNANCE_INVALID", "reference schema compatibility mismatch") + if value["fixtures"] != sorted(FIXTURES): + fail("REFERENCE_GOVERNANCE_INVALID", "reference fixture inventory mismatch") + if value["ordered_cases"] != list(ORDERED_CASES): + fail("REFERENCE_GOVERNANCE_INVALID", "reference ordered cases mismatch") + if value["performance_cases"] != list(PERFORMANCE_CASES): + fail("REFERENCE_GOVERNANCE_INVALID", "reference performance cases mismatch") + if value["execution_order"] != { + "warmups": list(WARMUP_ORDER), + "measured_pairs": [list(pair) for pair in FIVE_PAIR_ORDER], + }: + fail("REFERENCE_GOVERNANCE_INVALID", "reference execution order mismatch") + if value["pair_count"] != 5: + fail("REFERENCE_GOVERNANCE_INVALID", "reference pair count must be 5") + if not isinstance(value["threshold_policy_id"], str) or not value["threshold_policy_id"]: + fail("REFERENCE_GOVERNANCE_INVALID", "reference threshold policy ID is missing") + try: + raw_gate.require_sha256(value["threshold_policy_sha256"], "threshold policy digest") + except raw_gate.GateError as exc: + fail("REFERENCE_GOVERNANCE_INVALID", str(exc)) + return value + + +def verify_reference_governance( + manifest: dict[str, Any], *, reference_sha: str, candidate_sha: str, repository: pathlib.Path +) -> None: + validate_reference_manifest(manifest) + require_sha(reference_sha, "reference SHA") + require_sha(candidate_sha, "candidate SHA") + if manifest["reference_sha"] != reference_sha: + fail("REFERENCE_GOVERNANCE_INVALID", "effective reference differs from manifest") + for sha in (reference_sha, candidate_sha): + completed = subprocess.run( + [_git_executable(), "-C", str(repository), "cat-file", "-e", f"{sha}^{{commit}}"], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + fail("REFERENCE_GOVERNANCE_INVALID", f"commit {sha} is not reachable") + approval = manifest["approval"] + if approval["kind"] == "trusted_tag": + if not isinstance(approval["value"], str) or not re.fullmatch(r"v[0-9A-Za-z._-]+", approval["value"]): + fail("REFERENCE_GOVERNANCE_INVALID", "trusted tag name is invalid") + tagged = subprocess.run( + [_git_executable(), "-C", str(repository), "rev-parse", f"refs/tags/{approval['value']}^{{commit}}"], + capture_output=True, + text=True, + check=False, + ) + if tagged.returncode != 0 or tagged.stdout.strip() != reference_sha: + fail("REFERENCE_GOVERNANCE_INVALID", "trusted tag does not resolve to reference") + ancestor = subprocess.run( + [_git_executable(), "-C", str(repository), "merge-base", "--is-ancestor", reference_sha, candidate_sha], + capture_output=True, + text=True, + check=False, + ) + if ancestor.returncode != 0: + fail("REFERENCE_GOVERNANCE_INVALID", "reference is not an ancestor of candidate") + + +def reject_candidate_governance_changes(changed_paths: list[str]) -> None: + governed = { + "benchmarks/paired/reference-v1.13.json", + "benchmarks/paired/threshold-policy-v1.13.json", + } + if governed.intersection(changed_paths): + fail("REFERENCE_GOVERNANCE_INVALID", "ordinary candidate changes governed benchmark policy") + + +def _semantic_contract(row: dict[str, Any]) -> dict[str, Any]: + stats = row["execution_stats"] + return { + "case": row["case"], + "logical_files": stats["total_files"], + "logical_bytes": stats["total_bytes"], + "diagnostic_final_state": raw_gate.hard_final_state(row), + } + + +def validate_pair_inventory(records: list[dict[str, Any]], pair_count: int) -> None: + expected_order = measured_order(pair_count) + if len(records) != pair_count * 2: + fail("PAIR_INVENTORY_INVALID", "measured invocation count mismatch") + seen: set[tuple[int, str]] = set() + index = 0 + for ordinal, pair_order in enumerate(expected_order, start=1): + for position, side in enumerate(pair_order, start=1): + record = records[index] + index += 1 + pair_ordinal = record.get("pair_ordinal") + record_side = record.get("side") + if (pair_ordinal, record_side) in seen: + fail("PAIR_INVENTORY_INVALID", "duplicate paired invocation") + if ( + pair_ordinal != ordinal + or record.get("position") != position + or record_side != side + ): + fail("PAIR_INVENTORY_INVALID", f"altered invocation order at pair {ordinal}") + seen.add((ordinal, side)) + + +def validate_warmups( + warmups: list[dict[str, Any]], + measured: list[dict[str, Any]], + *, + dataset: str, + workers: int, + compression: str, +) -> None: + if len(warmups) != 2: + fail("PAIR_INVENTORY_INVALID", "warmup invocation count mismatch") + for position, side in enumerate(WARMUP_ORDER, start=1): + warmup = warmups[position - 1] + if ( + warmup.get("kind") != "warmup" + or warmup.get("pair_ordinal") is not None + or warmup.get("position") != position + or warmup.get("side") != side + ): + fail("PAIR_INVENTORY_INVALID", "warmup invocation order mismatch") + baseline_records = { + side: next(record for record in measured if record.get("side") == side) + for side in ("reference", "candidate") + } + baseline_rows = {} + baseline_data = {} + for side, record in baseline_records.items(): + data, rows = validate_raw_report( + record.get("envelope"), dataset=dataset, workers=workers, compression=compression + ) + baseline_data[side], baseline_rows[side] = data, rows + warmup_rows = {} + for warmup in warmups: + side = warmup["side"] + data, rows = validate_raw_report( + warmup.get("envelope"), dataset=dataset, workers=workers, compression=compression + ) + if ( + data["fixture"] != baseline_data[side]["fixture"] + or data["execution"] != baseline_data[side]["execution"] + ): + fail("EXECUTION_CONTRACT_MISMATCH", f"{side} warmup execution contract mismatch") + warmup_rows[side] = rows + for case_index, case_name in enumerate(ORDERED_CASES): + for side in ("reference", "candidate"): + if _semantic_contract(warmup_rows[side][case_index]) != _semantic_contract( + baseline_rows[side][case_index] + ): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{side} warmup hard state differs for {case_name}") + if _semantic_contract(warmup_rows["reference"][case_index]) != _semantic_contract( + warmup_rows["candidate"][case_index] + ): + fail("CORRECTNESS_REGRESSION", f"warmup hard state differs for {case_name}") + + +def _stability_boundary(mode: str, threshold: float | None) -> float: + if mode == "diagnostic": + return 2.5 + if threshold is None: + fail("CONTRACT_INVALID", "production comparison requires governed thresholds") + return min(3.0, threshold / 2.0) + + +def _decimal_median(values: list[Decimal]) -> Decimal: + ordered = sorted(values) + midpoint = len(ordered) // 2 + if len(ordered) % 2: + return ordered[midpoint] + return (ordered[midpoint - 1] + ordered[midpoint]) / Decimal(2) + + +def compare_records( + records: list[dict[str, Any]], + *, + pair_count: int, + dataset: str, + workers: int, + compression: str, + mode: str, + thresholds: dict[str, float] | None = None, +) -> dict[str, Any]: + if mode not in {"diagnostic", "production"}: + fail("CONTRACT_INVALID", f"unknown comparison mode {mode!r}") + if mode == "production" and pair_count != 5: + fail("PAIR_INVENTORY_INVALID", "production comparison requires exactly five pairs") + if mode == "diagnostic" and pair_count != 10: + fail("PAIR_INVENTORY_INVALID", "diagnostic comparison requires exactly ten pairs") + validate_pair_inventory(records, pair_count) + if mode == "production": + if thresholds is None or set(thresholds) != set(PERFORMANCE_CASES): + fail("CONTRACT_INVALID", "production threshold case set mismatch") + + validated: list[tuple[dict[str, Any], list[dict[str, Any]]]] = [] + first_fixture: dict[str, Any] | None = None + first_execution: dict[str, Any] | None = None + for index, record in enumerate(records, start=1): + data, rows = validate_raw_report( + record.get("envelope"), dataset=dataset, workers=workers, compression=compression + ) + if first_fixture is None: + first_fixture, first_execution = data["fixture"], data["execution"] + elif data["fixture"] != first_fixture or data["execution"] != first_execution: + fail("EXECUTION_CONTRACT_MISMATCH", f"fixture/execution changed in invocation {index}") + validated.append((data, rows)) + + case_results: list[dict[str, Any]] = [] + distributions: dict[str, dict[str, Any]] = {"reference": {}, "candidate": {}} + any_unstable = False + any_regression = False + any_diagnostic_rejection = False + for case_index, case_name in enumerate(ORDERED_CASES): + by_side: dict[str, list[dict[str, Any]]] = {"reference": [], "candidate": []} + for record, (_, rows) in zip(records, validated): + by_side[record["side"]].append(rows[case_index]) + for side, rows in by_side.items(): + expected = _semantic_contract(rows[0]) + if any(_semantic_contract(row) != expected for row in rows[1:]): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{side} hard state changed for {case_name}") + counters = [raw_gate.validate_operational_counters(row, workers=workers) for row in rows] + distributions[side][case_name] = raw_gate.summarize_operational_counters(counters) + + if _semantic_contract(by_side["reference"][0]) != _semantic_contract(by_side["candidate"][0]): + fail("CORRECTNESS_REGRESSION", f"reference/candidate hard state differs for {case_name}") + + if case_name not in PERFORMANCE_CASES: + case_results.append({"case": case_name, "performance_gated": False}) + continue + + ratio_values: list[Decimal] = [] + for ordinal in range(1, pair_count + 1): + pair_rows = { + record["side"]: rows[case_index] + for record, (_, rows) in zip(records, validated) + if record["pair_ordinal"] == ordinal + } + ratio_values.append( + Decimal(str(pair_rows["candidate"]["duration_ms"])) + / Decimal(str(pair_rows["reference"]["duration_ms"])) + ) + median_ratio_value = _decimal_median(ratio_values) + regression_value = (median_ratio_value - Decimal(1)) * Decimal(100) + mad_value = _decimal_median( + [abs(value - median_ratio_value) for value in ratio_values] + ) + mad_ratio_value = mad_value / median_ratio_value * Decimal(100) + ratios = [float(value) for value in ratio_values] + median_ratio = float(median_ratio_value) + regression_pct = float(regression_value) + paired_mad_ratio_pct = float(mad_ratio_value) + threshold = thresholds.get(case_name) if thresholds is not None else None + boundary = _stability_boundary(mode, threshold) + unstable = mad_ratio_value > Decimal(str(boundary)) + diagnostic_rejection = bool( + mode == "diagnostic" + and not Decimal("0.95") <= median_ratio_value <= Decimal("1.05") + ) + regression = bool( + mode == "production" + and not unstable + and threshold is not None + and regression_value > Decimal(str(threshold)) + ) + any_unstable = any_unstable or unstable + any_regression = any_regression or regression + any_diagnostic_rejection = any_diagnostic_rejection or diagnostic_rejection + logical_bytes = by_side["candidate"][0]["execution_stats"]["total_bytes"] + candidate_median_ms = float( + statistics.median(float(row["duration_ms"]) for row in by_side["candidate"]) + ) + case_results.append( + { + "case": case_name, + "performance_gated": True, + "paired_ratios": ratios, + "median_ratio": median_ratio, + "regression_pct": regression_pct, + "paired_mad_ratio_pct": paired_mad_ratio_pct, + "stability_boundary_pct": boundary, + "threshold_pct": threshold, + "candidate_throughput_mbps": ( + logical_bytes / (1024.0 * 1024.0) / (candidate_median_ms / 1000.0) + ), + "status": ( + "unstable" + if unstable + else "qualification_rejected" + if diagnostic_rejection + else "regression" + if regression + else "pass" + ), + } + ) + + classification = ( + "BENCHMARK_ENVIRONMENT_UNSTABLE" + if any_unstable + else "PERFORMANCE_REGRESSION" + if any_regression + else "DIAGNOSTIC_REJECTED" + if any_diagnostic_rejection + else "DIAGNOSTIC_QUALIFIED" + if mode == "diagnostic" + else "PASS" + ) + return { + "classification": classification, + "fixture": first_fixture, + "execution": first_execution, + "cases": case_results, + "operational_counter_distributions": distributions, + "hard_state_comparison": {"status": "equal", "case_count": len(ORDERED_CASES)}, + } + + +def _binary_hash(path: pathlib.Path) -> str: + return raw_gate.sha256_file(path) + + +def _host_observation() -> dict[str, Any]: + load = os.getloadavg() if hasattr(os, "getloadavg") else (0.0, 0.0, 0.0) + return { + "load_1m": load[0], + "load_5m": load[1], + "load_15m": load[2], + "cpu_count": os.cpu_count() or 0, + } + + +def _terminate_process_group(process: subprocess.Popen[str]) -> tuple[str, str]: + """Terminate and reap one owned benchmark process group.""" + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + return process.communicate(timeout=PROCESS_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return process.communicate() + + +def _remaining_profile_seconds(profile_deadline: float | None) -> float | None: + if profile_deadline is None: + return None + return profile_deadline - time.monotonic() + + +def _profile_elapsed_ms_or_fail(profile_started: float, mode: str) -> float: + elapsed_ms = (time.monotonic() - profile_started) * 1000.0 + if mode == "diagnostic" and elapsed_ms >= DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS: + fail( + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + "diagnostic profile did not finish validation within its fixed time budget", + ) + return elapsed_ms + + +def _capture( + *, + binary: pathlib.Path, + expected_hash: str, + side: str, + output_dir: pathlib.Path, + relative_raw_path: pathlib.Path, + dataset: str, + workers: int, + compression: str, + timeout_seconds: int, + profile_deadline: float | None = None, + profile_state: dict[str, Any] | None = None, + invocation: dict[str, Any] | None = None, +) -> dict[str, Any]: + if _binary_hash(binary) != expected_hash: + fail("BINARY_IDENTITY_INVALID", f"{side} binary changed during sampling") + raw_path = output_dir / relative_raw_path + stderr_path = raw_path.with_suffix(".stderr") + raw_path.parent.mkdir(parents=True, exist_ok=True) + before = _host_observation() + started = time.monotonic() + remaining = _remaining_profile_seconds(profile_deadline) + if remaining is not None and remaining <= 0: + if profile_state is not None: + profile_state["cancellation_reason"] = "internal profile deadline reached" + profile_state["active_invocation"] = invocation + fail( + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + "diagnostic profile exhausted its fixed time budget before the next invocation", + ) + effective_timeout = min(float(timeout_seconds), remaining) if remaining is not None else float(timeout_seconds) + deadline_limited = remaining is not None and remaining <= float(timeout_seconds) + command = [ + str(binary), + "benchmark", + "run", + "--dataset", + dataset, + "--workers", + str(workers), + "--repeat", + "1", + "--output", + "json", + ] + process: subprocess.Popen[str] | None = None + if profile_state is not None: + profile_state["active_invocation"] = invocation + try: + process = subprocess.Popen( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + env={**os.environ, "COLDKEEP_COMPRESSION": compression}, + ) + if profile_state is not None: + profile_state["process_started"] = True + try: + stdout, stderr = process.communicate(timeout=effective_timeout) + except subprocess.TimeoutExpired: + stdout, stderr = _terminate_process_group(process) + raw_path.write_text(stdout or "", encoding="utf-8") + stderr_path.write_text(stderr or "", encoding="utf-8") + if deadline_limited: + if profile_state is not None: + profile_state["cancellation_reason"] = "internal profile deadline reached" + fail( + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + "diagnostic profile exhausted its fixed time budget during an invocation", + ) + if side == "candidate": + fail("CANDIDATE_TIMEOUT_INCONCLUSIVE", "candidate command exceeded safety timeout") + fail("CI_INFRASTRUCTURE_TIMEOUT", "reference command exceeded safety timeout") + except OSError: + classification = ( + "REFERENCE_FUNCTIONAL_FAILURE" if side == "reference" else "CANDIDATE_FUNCTIONAL_FAILURE" + ) + fail(classification, f"{side} command could not be started") + except BaseException: + if process is not None and process.poll() is None: + stdout, stderr = _terminate_process_group(process) + raw_path.write_text(stdout or "", encoding="utf-8") + stderr_path.write_text(stderr or "", encoding="utf-8") + raise + + elapsed_ms = (time.monotonic() - started) * 1000.0 + raw_path.write_text(stdout, encoding="utf-8") + stderr_path.write_text(stderr, encoding="utf-8") + if process.returncode != 0: + classification = ( + "REFERENCE_FUNCTIONAL_FAILURE" if side == "reference" else "CANDIDATE_FUNCTIONAL_FAILURE" + ) + fail(classification, f"{side} command failed with exit {process.returncode}") + try: + envelope = load_json_strict(raw_path) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + validate_raw_report(envelope, dataset=dataset, workers=workers, compression=compression) + if profile_state is not None: + profile_state["active_invocation"] = None + return { + "envelope": envelope, + "raw_file": relative_raw_path.as_posix(), + "stderr_file": relative_raw_path.with_suffix(".stderr").as_posix(), + "command_duration_ms": elapsed_ms, + "binary_sha256": expected_hash, + "host_observation": {"before": before, "after": _host_observation()}, + } + + +def _write_checksums(directory: pathlib.Path) -> None: + lines = [] + for relative in sorted(_artifact_files(directory)): + if relative == "checksums.sha256": + continue + path = _contained_regular_file( + directory, pathlib.PurePosixPath(relative), f"artifact file {relative}" + ) + lines.append(f"{_binary_hash(path)} {relative}") + (directory / "checksums.sha256").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def validate_checksums( + directory: pathlib.Path, *, expected_files: set[str] | None = None +) -> None: + checksum_path = directory / "checksums.sha256" + if checksum_path.is_symlink() or not checksum_path.is_file(): + fail("EVIDENCE_INTEGRITY_FAILURE", "artifact checksum inventory is missing") + actual_files = _artifact_files(directory) - {"checksums.sha256"} + seen: set[str] = set() + for line in checksum_path.read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([^\r\n]+)", line) + if match is None: + fail("EVIDENCE_INTEGRITY_FAILURE", "malformed checksum inventory") + digest, relative = match.groups() + normalized = require_relative_artifact_path(relative, "checksum path") + if relative in seen: + fail("EVIDENCE_INTEGRITY_FAILURE", "unsafe or duplicate checksum path") + seen.add(relative) + path = _contained_regular_file(directory, normalized, f"checksummed file {relative}") + if _binary_hash(path) != digest: + fail("EVIDENCE_INTEGRITY_FAILURE", f"checksum mismatch for {relative}") + if seen != actual_files: + fail("EVIDENCE_INTEGRITY_FAILURE", "checksum inventory coverage mismatch") + if expected_files is not None and actual_files != expected_files: + fail("EVIDENCE_INTEGRITY_FAILURE", "artifact file inventory mismatch") + + +def _provenance(args: argparse.Namespace) -> dict[str, Any]: + return { + "event_name": os.environ.get("GITHUB_EVENT_NAME", "local"), + "repository_id": os.environ.get("GITHUB_REPOSITORY", "local"), + "runner_os": os.environ.get("RUNNER_OS", sys.platform), + "runner_image": os.environ.get("ImageVersion", "local"), + "runner_arch": os.environ.get("RUNNER_ARCH", platform.machine()), + "cpu_count": os.cpu_count() or 0, + "go_version": args.go_version, + "postgres_version": args.postgres_version, + "database_image_digest": args.database_image_digest, + } + + +def _benchmark_temp_roots() -> set[pathlib.Path]: + temp_root = pathlib.Path(tempfile.gettempdir()) + return { + path.resolve() + for path in temp_root.glob("coldkeep-benchmark-*") + if path.is_dir() and not path.is_symlink() + } + + +def _cleanup_benchmark_databases() -> tuple[int, int]: + required = ("DB_HOST", "DB_PORT", "DB_USER") + if any(not os.environ.get(name) for name in required) or shutil.which("psql") is None: + return 0, 1 + command = [ + "psql", + "-X", + "--no-psqlrc", + "--tuples-only", + "--no-align", + "--set", + "ON_ERROR_STOP=1", + "--host", + os.environ["DB_HOST"], + "--port", + os.environ["DB_PORT"], + "--username", + os.environ["DB_USER"], + "--dbname", + os.environ.get("COLDKEEP_TEST_DB_MAINTENANCE", "postgres"), + ] + psql_env = dict(os.environ) + if os.environ.get("DB_PASSWORD"): + psql_env["PGPASSWORD"] = os.environ["DB_PASSWORD"] + query = subprocess.run( + [*command, "--command", "SELECT datname FROM pg_database WHERE datname LIKE 'coldkeep\\_bench\\_%' ESCAPE '\\';"], + text=True, + capture_output=True, + env=psql_env, + check=False, + ) + if query.returncode != 0: + return 0, 1 + names = [line.strip() for line in query.stdout.splitlines() if line.strip()] + if any(re.fullmatch(r"coldkeep_bench_[a-z0-9_]+", name) is None for name in names): + return 0, 1 + removed = 0 + errors = 0 + for name in names: + quoted = '"' + name.replace('"', '""') + '"' + terminate_statement = ( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + f"WHERE datname = '{name}' AND pid <> pg_backend_pid();" + ) + completed = subprocess.run( + [ + *command, + "--command", + terminate_statement, + "--command", + f"DROP DATABASE IF EXISTS {quoted};", + ], + text=True, + capture_output=True, + env=psql_env, + check=False, + ) + if completed.returncode == 0: + removed += 1 + else: + errors += 1 + return removed, errors + + +def _cleanup_interrupted_profile(args: argparse.Namespace) -> dict[str, Any]: + state = getattr(args, "_profile_state", {}) + before = state.get("temp_roots_before", set()) + removed_roots = 0 + errors = 0 + for path in sorted(_benchmark_temp_roots() - before): + try: + shutil.rmtree(path) + removed_roots += 1 + except OSError: + errors += 1 + removed_databases = 0 + if state.get("process_started") and state.get("active_invocation") is not None: + removed_databases, database_errors = _cleanup_benchmark_databases() + errors += database_errors + return { + "status": "complete" if errors == 0 else "incomplete", + "filesystem_entries_removed": removed_roots, + "databases_removed": removed_databases, + "errors": errors, + } + + +def sample_command(args: argparse.Namespace) -> int: + profile_started = time.monotonic() + profile_deadline = profile_started + DIAGNOSTIC_MAX_PROFILE_ELAPSED_SECONDS + args._profile_state = { + "started": profile_started, + "deadline": profile_deadline, + "active_invocation": None, + "cancellation_reason": None, + "process_started": False, + "temp_roots_before": _benchmark_temp_roots(), + } + fixture_contract(args.dataset, args.workers) + if (args.mode, args.pairs) not in {("diagnostic", 10), ("production", 5)}: + fail("PAIR_INVENTORY_INVALID", "mode and fixed pair count are inconsistent") + if args.command_timeout_seconds != 600: + fail("EXECUTION_CONTRACT_MISMATCH", "per-command safety timeout must be 600 seconds") + if os.environ.get("COLDKEEP_CODEC") != "aes-gcm": + fail("EXECUTION_CONTRACT_MISMATCH", "paired sampling requires COLDKEEP_CODEC=aes-gcm") + if not args.reference_binary.is_file() or not args.candidate_binary.is_file(): + fail("BINARY_IDENTITY_INVALID", "reference and candidate binaries must exist") + candidate_sha = require_sha(args.candidate_sha, "candidate SHA") + thresholds: dict[str, float] | None = None + governance: dict[str, Any] + if args.mode == "production": + if args.reference_sha is not None: + fail( + "REFERENCE_GOVERNANCE_INVALID", + "production reference SHA cannot come from command input", + ) + if not PRODUCTION_SAMPLING_AUTHORIZED: + fail( + "REFERENCE_GOVERNANCE_INVALID", + "production paired sampling is not authorized in this repository state", + ) + repository = _repository_root() + manifest_path = _governed_repository_file( + repository, GOVERNED_MANIFEST_RELATIVE, "governed reference manifest" + ) + threshold_path = _governed_repository_file( + repository, GOVERNED_THRESHOLD_RELATIVE, "governed threshold policy" + ) + manifest = validate_reference_manifest(load_json_strict(manifest_path)) + reference_sha = require_sha(manifest.get("reference_sha"), "reference SHA") + verify_reference_governance( + manifest, + reference_sha=reference_sha, + candidate_sha=candidate_sha, + repository=repository, + ) + policy_value = load_json_strict(threshold_path) + thresholds = validate_threshold_policy(policy_value) + policy_hash = _binary_hash(threshold_path) + if manifest["threshold_policy_sha256"] != policy_hash: + fail("REFERENCE_GOVERNANCE_INVALID", "threshold policy digest differs from manifest") + args.reference_sha = reference_sha + governance_dir = args.output_dir / "governance" + governance_dir.mkdir(parents=False, exist_ok=False) + shutil.copyfile(manifest_path, governance_dir / "reference-manifest.json") + shutil.copyfile(threshold_path, governance_dir / "threshold-policy.json") + governance = { + "status": "governed", + "manifest_sha256": _binary_hash(manifest_path), + "threshold_policy_id": manifest["threshold_policy_id"], + "threshold_policy_sha256": policy_hash, + } + else: + if args.reference_sha is None: + fail("REFERENCE_GOVERNANCE_INVALID", "diagnostic sampling requires an explicit reference SHA") + reference_sha = require_sha(args.reference_sha, "reference SHA") + governance = { + "status": "provisional-diagnostic", + "manifest_sha256": None, + "threshold_policy_id": None, + "threshold_policy_sha256": None, + } + + binary_hashes = { + "reference": _binary_hash(args.reference_binary), + "candidate": _binary_hash(args.candidate_binary), + } + if args.mode == "diagnostic" and binary_hashes["reference"] != binary_hashes["candidate"]: + fail( + "BINARY_IDENTITY_INVALID", + "diagnostic qualification requires byte-identical reference and candidate binaries", + ) + warmup_records: list[dict[str, Any]] = [] + for position, side in enumerate(WARMUP_ORDER, start=1): + invocation = {"kind": "warmup", "pair_ordinal": None, "position": position, "side": side} + record = _capture( + binary=getattr(args, f"{side}_binary"), + expected_hash=binary_hashes[side], + side=side, + output_dir=args.output_dir, + relative_raw_path=pathlib.Path("raw") / f"warmup-{position:02d}-{side}.json", + dataset=args.dataset, + workers=args.workers, + compression=args.compression, + timeout_seconds=args.command_timeout_seconds, + profile_deadline=profile_deadline if args.mode == "diagnostic" else None, + profile_state=args._profile_state, + invocation=invocation, + ) + record.update(invocation) + warmup_records.append(record) + + records: list[dict[str, Any]] = [] + for ordinal, pair_order in enumerate(measured_order(args.pairs), start=1): + for position, side in enumerate(pair_order, start=1): + invocation = { + "kind": "measured", + "pair_ordinal": ordinal, + "position": position, + "side": side, + } + record = _capture( + binary=getattr(args, f"{side}_binary"), + expected_hash=binary_hashes[side], + side=side, + output_dir=args.output_dir, + relative_raw_path=( + pathlib.Path("raw") / f"pair-{ordinal:02d}" / f"{position:02d}-{side}.json" + ), + dataset=args.dataset, + workers=args.workers, + compression=args.compression, + timeout_seconds=args.command_timeout_seconds, + profile_deadline=profile_deadline if args.mode == "diagnostic" else None, + profile_state=args._profile_state, + invocation=invocation, + ) + record.update(invocation) + records.append(record) + + comparison = compare_records( + records, + pair_count=args.pairs, + dataset=args.dataset, + workers=args.workers, + compression=args.compression, + mode=args.mode, + thresholds=thresholds, + ) + validate_warmups( + warmup_records, + records, + dataset=args.dataset, + workers=args.workers, + compression=args.compression, + ) + try: + profile_elapsed_ms = _profile_elapsed_ms_or_fail(profile_started, args.mode) + except PairedGateError: + args._profile_state["cancellation_reason"] = "internal profile deadline reached" + raise + classification = comparison["classification"] + inventory = [] + for record in warmup_records + records: + inventory.append( + { + key: record[key] + for key in ( + "kind", + "pair_ordinal", + "position", + "side", + "raw_file", + "stderr_file", + "command_duration_ms", + "binary_sha256", + "host_observation", + ) + } + ) + report = { + "schema_version": SCHEMA_VERSION, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": REPORT_KIND, + "status": "complete", + "mode": args.mode, + "classification": classification, + "contract_version": CONTRACT_VERSION, + "authority": authority_contract(args.mode), + "identity": { + "reference_sha": reference_sha, + "candidate_sha": candidate_sha, + "reference_binary_sha256": binary_hashes["reference"], + "candidate_binary_sha256": binary_hashes["candidate"], + }, + "governance": governance, + "profile": { + "codec": "aes-gcm", + "compression": args.compression, + "dataset": args.dataset, + "workers": args.workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "fixture": comparison["fixture"], + "warmup_order": list(WARMUP_ORDER), + "measured_order": [list(pair) for pair in measured_order(args.pairs)], + "pair_count": args.pairs, + "profile_elapsed_ms": profile_elapsed_ms, + "invocation_inventory": inventory, + "cases": comparison["cases"], + "operational_counter_distributions": comparison["operational_counter_distributions"], + "hard_state_comparison": comparison["hard_state_comparison"], + "cleanup": { + "status": "complete", + "attempted": (2 + args.pairs * 2) * len(ORDERED_CASES), + "succeeded": (2 + args.pairs * 2) * len(ORDERED_CASES), + "failed": 0, + }, + "provenance": _provenance(args), + } + raw_gate.validate_no_sensitive_evidence(report, "paired comparison report") + try: + report["profile_elapsed_ms"] = _profile_elapsed_ms_or_fail(profile_started, args.mode) + except PairedGateError: + args._profile_state["cancellation_reason"] = "internal profile deadline reached" + raise + raw_gate.write_json(args.output_dir / "paired-comparison.json", report) + _write_checksums(args.output_dir) + print(json.dumps({"classification": report["classification"], "report": "paired-comparison.json"})) + return 0 if report["classification"] in SUCCESS_CLASSIFICATIONS else 1 + + +REPORT_FIELDS = { + "schema_version", + "evidence_policy_version", + "report_kind", + "status", + "mode", + "classification", + "contract_version", + "authority", + "identity", + "governance", + "profile", + "fixture", + "warmup_order", + "measured_order", + "pair_count", + "profile_elapsed_ms", + "invocation_inventory", + "cases", + "operational_counter_distributions", + "hard_state_comparison", + "cleanup", + "provenance", +} + +FAILURE_REPORT_FIELDS = { + "schema_version", + "evidence_policy_version", + "report_kind", + "status", + "mode", + "classification", + "contract_version", + "authority", + "identity", + "governance_status", + "profile", + "requested_pair_count", + "warmup_order", + "measured_order", + "attempted_invocations", + "active_invocation", + "profile_elapsed_ms", + "cancellation", + "prefix_validation", + "cleanup", + "provenance", +} + + +def _validate_failure_report(report: Any, *, expected_profile: str | None) -> dict[str, Any]: + try: + report = raw_gate.require_exact_fields(report, FAILURE_REPORT_FIELDS, "paired failure report") + identity = raw_gate.require_exact_fields( + report["identity"], + { + "reference_sha", + "candidate_sha", + "reference_binary_sha256", + "candidate_binary_sha256", + }, + "paired failure identity", + ) + profile = raw_gate.require_exact_fields( + report["profile"], + {"codec", "compression", "dataset", "workers", "pipeline_depth", "deterministic"}, + "paired failure profile", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if ( + report["schema_version"] != 1 + or report["evidence_policy_version"] != 2 + or report["report_kind"] != REPORT_KIND + or report["status"] != "failed" + or report["contract_version"] != CONTRACT_VERSION + or report["classification"] + not in CLASSIFICATIONS + - SUCCESS_CLASSIFICATIONS + - {"PERFORMANCE_REGRESSION", "BENCHMARK_ENVIRONMENT_UNSTABLE"} + ): + fail("CONTRACT_INVALID", "paired failure report identity mismatch") + if report["mode"] not in DECISION_MODES or report["authority"] != authority_contract( + report["mode"] + ): + fail("REFERENCE_GOVERNANCE_INVALID", "paired failure authority mismatch") + if expected_profile is not None: + expected = PROFILE_MATRIX.get(expected_profile) + if expected is None or (profile["compression"], profile["workers"], profile["dataset"]) != expected: + fail("EXECUTION_CONTRACT_MISMATCH", f"failure profile {expected_profile} mismatch") + if profile["codec"] != "aes-gcm" or profile["pipeline_depth"] != 1 or profile["deterministic"] is not True: + fail("EXECUTION_CONTRACT_MISMATCH", "paired failure profile policy mismatch") + if report["requested_pair_count"] not in {5, 10}: + fail("PAIR_INVENTORY_INVALID", "failure report pair count is invalid") + if report["warmup_order"] != list(WARMUP_ORDER) or report["measured_order"] != [ + list(pair) for pair in measured_order(report["requested_pair_count"]) + ]: + fail("PAIR_INVENTORY_INVALID", "failure report execution order mismatch") + expected_governance = ( + "provisional-diagnostic" if report["mode"] == "diagnostic" else "not-established" + ) + if report["governance_status"] != expected_governance: + fail("REFERENCE_GOVERNANCE_INVALID", "failure governance status mismatch") + attempted = report["attempted_invocations"] + if not isinstance(attempted, list): + fail("PAIR_INVENTORY_INVALID", "failure attempted inventory must be an array") + expected_attempts = [ + {"kind": "warmup", "pair_ordinal": None, "position": position, "side": side} + for position, side in enumerate(WARMUP_ORDER, start=1) + ] + expected_attempts.extend( + {"kind": "measured", "pair_ordinal": ordinal, "position": position, "side": side} + for ordinal, pair_order in enumerate(measured_order(report["requested_pair_count"]), start=1) + for position, side in enumerate(pair_order, start=1) + ) + normalized_attempts = [] + try: + for index, invocation in enumerate(attempted): + invocation = raw_gate.require_exact_fields( + invocation, + {"kind", "pair_ordinal", "position", "side", "raw_file", "stderr_file"}, + f"failure invocation {index + 1}", + ) + normalized_attempts.append( + {key: invocation[key] for key in ("kind", "pair_ordinal", "position", "side")} + ) + except raw_gate.GateError as exc: + fail("PAIR_INVENTORY_INVALID", str(exc)) + if normalized_attempts != expected_attempts[: len(normalized_attempts)]: + fail("PAIR_INVENTORY_INVALID", "failure invocation inventory is not a fixed-order prefix") + for index, (invocation, expected) in enumerate( + zip(attempted, expected_attempts), start=1 + ): + if expected["kind"] == "warmup": + expected_raw = f"raw/warmup-{expected['position']:02d}-{expected['side']}.json" + else: + expected_raw = ( + f"raw/pair-{expected['pair_ordinal']:02d}/" + f"{expected['position']:02d}-{expected['side']}.json" + ) + if ( + require_relative_artifact_path( + invocation["raw_file"], f"failure invocation {index} raw file" + ).as_posix() + != expected_raw + or require_relative_artifact_path( + invocation["stderr_file"], f"failure invocation {index} stderr file" + ).as_posix() + != pathlib.PurePosixPath(expected_raw).with_suffix(".stderr").as_posix() + ): + fail("PAIR_INVENTORY_INVALID", "failure invocation path/order mismatch") + try: + cleanup = raw_gate.require_exact_fields( + report["cleanup"], + { + "status", + "observed_invocations", + "required_invocations", + "completed_cases", + "active_invocation", + "filesystem_entries_removed", + "databases_removed", + "errors", + }, + "failure cleanup", + ) + observed_invocations = raw_gate.require_nonnegative_integer( + cleanup["observed_invocations"], "failure observed invocations" + ) + required_invocations = raw_gate.require_nonnegative_integer( + cleanup["required_invocations"], "failure required invocations" + ) + completed_cases = raw_gate.require_nonnegative_integer( + cleanup["completed_cases"], "failure completed cases" + ) + for field in ("filesystem_entries_removed", "databases_removed", "errors"): + raw_gate.require_nonnegative_integer(cleanup[field], f"failure cleanup {field}") + except raw_gate.GateError as exc: + fail("EVIDENCE_INTEGRITY_FAILURE", str(exc)) + if ( + cleanup["status"] not in {"complete", "incomplete"} + or observed_invocations != len(attempted) + or required_invocations != len(expected_attempts) + or observed_invocations >= required_invocations + or completed_cases != observed_invocations * len(ORDERED_CASES) + or cleanup["active_invocation"] not in {"not_applicable", "cleaned", "incomplete"} + ): + fail("EVIDENCE_INTEGRITY_FAILURE", "failure cleanup/inventory evidence mismatch") + if cleanup["status"] == "complete" and cleanup["errors"] != 0: + fail("EVIDENCE_INTEGRITY_FAILURE", "complete cleanup reports errors") + if cleanup["status"] == "incomplete" and cleanup["errors"] == 0: + fail("EVIDENCE_INTEGRITY_FAILURE", "incomplete cleanup lacks an error") + try: + raw_gate.require_number(report["profile_elapsed_ms"], "failure profile elapsed", positive=True) + cancellation = raw_gate.require_exact_fields( + report["cancellation"], {"reason", "authoritative"}, "failure cancellation" + ) + prefix = raw_gate.require_exact_fields( + report["prefix_validation"], + {"status", "raw_report_count", "case_row_count", "counter_validation", "hard_state"}, + "failure prefix validation", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if ( + not isinstance(cancellation["reason"], str) + or not cancellation["reason"] + or cancellation["authoritative"] is not False + or prefix["status"] not in {"validated", "not_evaluated"} + or prefix["raw_report_count"] != len(attempted) + or prefix["case_row_count"] != len(attempted) * len(ORDERED_CASES) + or prefix["counter_validation"] not in {"valid", "not_evaluated"} + or prefix["hard_state"] not in {"equal", "mismatch", "not_evaluated"} + ): + fail("CONTRACT_INVALID", "failure cancellation/prefix evidence mismatch") + active = report["active_invocation"] + if active is not None: + try: + active = raw_gate.require_exact_fields( + active, + { + "kind", + "pair_ordinal", + "position", + "side", + "raw_file", + "stderr_file", + "raw_capture_present", + "stderr_capture_present", + "capture_validation", + "status", + }, + "active invocation", + ) + except raw_gate.GateError as exc: + fail("PAIR_INVENTORY_INVALID", str(exc)) + if ( + active["status"] != "incomplete" + or active["capture_validation"] != "unvalidated" + or active["side"] not in {"reference", "candidate"} + or not isinstance(active["raw_capture_present"], bool) + or not isinstance(active["stderr_capture_present"], bool) + ): + fail("PAIR_INVENTORY_INVALID", "active invocation status is invalid") + if len(attempted) >= len(expected_attempts): + fail("PAIR_INVENTORY_INVALID", "complete failure inventory cannot have an active invocation") + next_expected = expected_attempts[len(attempted)] + if any(active[field] != next_expected[field] for field in ("kind", "pair_ordinal", "position", "side")): + fail("PAIR_INVENTORY_INVALID", "active invocation is not next in fixed order") + if next_expected["kind"] == "warmup": + expected_active_raw = ( + f"raw/warmup-{next_expected['position']:02d}-{next_expected['side']}.json" + ) + else: + expected_active_raw = ( + f"raw/pair-{next_expected['pair_ordinal']:02d}/" + f"{next_expected['position']:02d}-{next_expected['side']}.json" + ) + if ( + require_relative_artifact_path(active["raw_file"], "active raw file").as_posix() + != expected_active_raw + or require_relative_artifact_path( + active["stderr_file"], "active stderr file" + ).as_posix() + != pathlib.PurePosixPath(expected_active_raw).with_suffix(".stderr").as_posix() + ): + fail("PAIR_INVENTORY_INVALID", "active invocation path mismatch") + for field in ("reference_sha", "candidate_sha"): + if not isinstance(identity[field], str) or not re.fullmatch(r"[0-9a-f]{40}", identity[field]): + if report["classification"] != "REFERENCE_GOVERNANCE_INVALID": + fail("CONTRACT_INVALID", f"paired failure {field} is invalid") + for field in ("reference_binary_sha256", "candidate_binary_sha256"): + if identity[field] is not None: + try: + raw_gate.require_sha256(identity[field], f"paired failure {field}") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + try: + provenance = raw_gate.require_exact_fields( + report["provenance"], + { + "event_name", + "repository_id", + "runner_os", + "runner_image", + "runner_arch", + "cpu_count", + "go_version", + "postgres_version", + "database_image_digest", + }, + "paired failure provenance", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + validate_repository_id(provenance["repository_id"]) + raw_gate.validate_no_sensitive_evidence(report, "paired failure report") + return report + + +def validate_report_summary(report: Any, *, expected_profile: str | None = None) -> dict[str, Any]: + if isinstance(report, dict) and report.get("status") == "failed": + return _validate_failure_report(report, expected_profile=expected_profile) + try: + report = raw_gate.require_exact_fields(report, REPORT_FIELDS, "paired comparison report") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if ( + report["schema_version"] != 1 + or report["evidence_policy_version"] != 2 + or report["report_kind"] != REPORT_KIND + or report["status"] != "complete" + or report["contract_version"] != CONTRACT_VERSION + or report["classification"] not in CLASSIFICATIONS + ): + fail("CONTRACT_INVALID", "paired comparison report identity mismatch") + if report["mode"] not in DECISION_MODES or report["authority"] != authority_contract( + report["mode"] + ): + fail("REFERENCE_GOVERNANCE_INVALID", "paired report authority mismatch") + profile = report["profile"] + try: + profile = raw_gate.require_exact_fields( + profile, + {"codec", "compression", "dataset", "workers", "pipeline_depth", "deterministic"}, + "paired profile", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if expected_profile is not None: + expected = PROFILE_MATRIX.get(expected_profile) + if expected is None or (profile["compression"], profile["workers"], profile["dataset"]) != expected: + fail("EXECUTION_CONTRACT_MISMATCH", f"profile {expected_profile} identity mismatch") + if ( + profile["codec"] != "aes-gcm" + or profile["compression"] not in {"none", "zstd"} + or profile["pipeline_depth"] != 1 + or profile["deterministic"] is not True + ): + fail("EXECUTION_CONTRACT_MISMATCH", "paired profile policy mismatch") + validate_fixture(report["fixture"], dataset=profile["dataset"], workers=profile["workers"]) + + try: + identity = raw_gate.require_exact_fields( + report["identity"], + { + "reference_sha", + "candidate_sha", + "reference_binary_sha256", + "candidate_binary_sha256", + }, + "paired identity", + ) + raw_gate.require_sha256(identity["reference_binary_sha256"], "reference binary") + raw_gate.require_sha256(identity["candidate_binary_sha256"], "candidate binary") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + for field in ("reference_sha", "candidate_sha"): + if not isinstance(identity[field], str) or not re.fullmatch(r"[0-9a-f]{40}", identity[field]): + fail("CONTRACT_INVALID", f"paired identity {field} is invalid") + if ( + report["mode"] == "diagnostic" + and identity["reference_binary_sha256"] != identity["candidate_binary_sha256"] + ): + fail("BINARY_IDENTITY_INVALID", "diagnostic binaries are not byte-identical") + try: + governance = raw_gate.require_exact_fields( + report["governance"], + { + "status", + "manifest_sha256", + "threshold_policy_id", + "threshold_policy_sha256", + }, + "paired governance", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if report["mode"] == "production": + if governance["status"] != "governed": + fail("REFERENCE_GOVERNANCE_INVALID", "production report is not governed") + try: + raw_gate.require_sha256(governance["manifest_sha256"], "manifest digest") + raw_gate.require_sha256(governance["threshold_policy_sha256"], "threshold digest") + except raw_gate.GateError as exc: + fail("REFERENCE_GOVERNANCE_INVALID", str(exc)) + if not isinstance(governance["threshold_policy_id"], str) or not governance["threshold_policy_id"]: + fail("REFERENCE_GOVERNANCE_INVALID", "threshold policy identity is missing") + elif report["mode"] == "diagnostic": + if governance != { + "status": "provisional-diagnostic", + "manifest_sha256": None, + "threshold_policy_id": None, + "threshold_policy_sha256": None, + }: + fail("REFERENCE_GOVERNANCE_INVALID", "diagnostic governance fields mismatch") + else: + fail("CONTRACT_INVALID", "paired report mode is invalid") + if report["warmup_order"] != list(WARMUP_ORDER): + fail("PAIR_INVENTORY_INVALID", "warmup order mismatch") + if (report["mode"], report["pair_count"]) not in {("diagnostic", 10), ("production", 5)}: + fail("PAIR_INVENTORY_INVALID", "report mode and pair count mismatch") + try: + profile_elapsed_ms = raw_gate.require_number( + report["profile_elapsed_ms"], "profile elapsed duration", positive=True + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if report["measured_order"] != [list(pair) for pair in measured_order(report["pair_count"])]: + fail("PAIR_INVENTORY_INVALID", "measured order mismatch") + inventory = report["invocation_inventory"] + if not isinstance(inventory, list) or len(inventory) != 2 + report["pair_count"] * 2: + fail("PAIR_INVENTORY_INVALID", "report invocation inventory count mismatch") + inventory_fields = { + "kind", + "pair_ordinal", + "position", + "side", + "raw_file", + "stderr_file", + "command_duration_ms", + "binary_sha256", + "host_observation", + } + try: + for index, invocation in enumerate(inventory): + raw_gate.require_exact_fields(invocation, inventory_fields, f"invocation {index + 1}") + raw_gate.require_number( + invocation["command_duration_ms"], f"invocation {index + 1} duration", positive=True + ) + raw_gate.require_sha256(invocation["binary_sha256"], f"invocation {index + 1} binary") + host = raw_gate.require_exact_fields( + invocation["host_observation"], {"before", "after"}, f"invocation {index + 1} host" + ) + for point in ("before", "after"): + values = raw_gate.require_exact_fields( + host[point], + {"load_1m", "load_5m", "load_15m", "cpu_count"}, + f"invocation {index + 1} host {point}", + ) + for field in ("load_1m", "load_5m", "load_15m", "cpu_count"): + raw_gate.require_number(values[field], f"invocation {index + 1} {point} {field}") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + expected_inventory_paths: list[tuple[str, str]] = [] + for position, side in enumerate(WARMUP_ORDER, start=1): + raw_file = f"raw/warmup-{position:02d}-{side}.json" + expected_inventory_paths.append( + (raw_file, pathlib.PurePosixPath(raw_file).with_suffix(".stderr").as_posix()) + ) + for ordinal, pair_order in enumerate(measured_order(report["pair_count"]), start=1): + for position, side in enumerate(pair_order, start=1): + raw_file = f"raw/pair-{ordinal:02d}/{position:02d}-{side}.json" + expected_inventory_paths.append( + (raw_file, pathlib.PurePosixPath(raw_file).with_suffix(".stderr").as_posix()) + ) + for index, (invocation, expected_paths) in enumerate( + zip(inventory, expected_inventory_paths), start=1 + ): + raw_file = require_relative_artifact_path( + invocation["raw_file"], f"invocation {index} raw file" + ).as_posix() + stderr_file = require_relative_artifact_path( + invocation["stderr_file"], f"invocation {index} stderr file" + ).as_posix() + if (raw_file, stderr_file) != expected_paths: + fail("PAIR_INVENTORY_INVALID", f"invocation {index} artifact path mismatch") + for position, side in enumerate(WARMUP_ORDER, start=1): + invocation = inventory[position - 1] + if ( + invocation["kind"] != "warmup" + or invocation["pair_ordinal"] is not None + or invocation["position"] != position + or invocation["side"] != side + ): + fail("PAIR_INVENTORY_INVALID", "warmup inventory mismatch") + measured_inventory = [ + { + "pair_ordinal": invocation["pair_ordinal"], + "position": invocation["position"], + "side": invocation["side"], + } + for invocation in inventory[2:] + ] + validate_pair_inventory(measured_inventory, report["pair_count"]) + for invocation in inventory: + expected_hash = identity[f"{invocation['side']}_binary_sha256"] + if invocation["binary_sha256"] != expected_hash: + fail("BINARY_IDENTITY_INVALID", "invocation binary hash continuity failure") + + cases = report["cases"] + if not isinstance(cases, list) or [case.get("case") for case in cases if isinstance(case, dict)] != list(ORDERED_CASES): + fail("CONTRACT_INVALID", "paired report case set/order mismatch") + any_unstable = False + any_regression = False + any_diagnostic_rejection = False + for case in cases: + case_name = case["case"] + if case_name not in PERFORMANCE_CASES: + try: + raw_gate.require_exact_fields(case, {"case", "performance_gated"}, f"case {case_name}") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if case["performance_gated"] is not False: + fail("CONTRACT_INVALID", f"case {case_name} performance policy mismatch") + continue + expected_case_fields = { + "case", + "performance_gated", + "paired_ratios", + "median_ratio", + "regression_pct", + "paired_mad_ratio_pct", + "stability_boundary_pct", + "threshold_pct", + "candidate_throughput_mbps", + "status", + } + try: + raw_gate.require_exact_fields(case, expected_case_fields, f"case {case_name}") + if not isinstance(case["paired_ratios"], list) or len(case["paired_ratios"]) != report["pair_count"]: + fail("CONTRACT_INVALID", f"case {case_name} ratio count mismatch") + ratio_values = [ + Decimal(str(raw_gate.require_number(value, f"case {case_name} ratio", positive=True))) + for value in case["paired_ratios"] + ] + median_value = _decimal_median(ratio_values) + mad_value = _decimal_median([abs(value - median_value) for value in ratio_values]) + expected_regression = (median_value - Decimal(1)) * Decimal(100) + expected_mad = mad_value / median_value * Decimal(100) + for field in ("median_ratio", "stability_boundary_pct", "candidate_throughput_mbps"): + raw_gate.require_number( + case[field], f"case {case_name} {field}", positive=True + ) + raw_gate.require_number( + case["paired_mad_ratio_pct"], f"case {case_name} paired_mad_ratio_pct" + ) + regression_number = case["regression_pct"] + if ( + isinstance(regression_number, bool) + or not isinstance(regression_number, (int, float)) + or not math.isfinite(float(regression_number)) + ): + fail("CONTRACT_INVALID", f"case {case_name} regression_pct must be finite numeric") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if not math.isclose(case["median_ratio"], float(median_value), rel_tol=1e-12, abs_tol=1e-12): + fail("CONTRACT_INVALID", f"case {case_name} median ratio mismatch") + if not math.isclose(case["regression_pct"], float(expected_regression), rel_tol=1e-12, abs_tol=1e-12): + fail("CONTRACT_INVALID", f"case {case_name} regression mismatch") + if not math.isclose(case["paired_mad_ratio_pct"], float(expected_mad), rel_tol=1e-12, abs_tol=1e-12): + fail("CONTRACT_INVALID", f"case {case_name} paired MAD mismatch") + if report["mode"] == "diagnostic": + if case["threshold_pct"] is not None or case["stability_boundary_pct"] != 2.5: + fail("CONTRACT_INVALID", f"case {case_name} diagnostic threshold fields mismatch") + unstable = expected_mad > Decimal("2.5") + diagnostic_rejection = not Decimal("0.95") <= median_value <= Decimal("1.05") + regression = False + else: + try: + threshold = raw_gate.require_number( + case["threshold_pct"], f"case {case_name} threshold", positive=True + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if threshold > 10: + fail("CONTRACT_INVALID", f"case {case_name} threshold exceeds 10%") + expected_boundary = min(3.0, threshold / 2.0) + if case["stability_boundary_pct"] != expected_boundary: + fail("CONTRACT_INVALID", f"case {case_name} stability boundary mismatch") + unstable = expected_mad > Decimal(str(expected_boundary)) + diagnostic_rejection = False + regression = not unstable and expected_regression > Decimal(str(threshold)) + expected_status = ( + "unstable" + if unstable + else "qualification_rejected" + if diagnostic_rejection + else "regression" + if regression + else "pass" + ) + if case["status"] != expected_status: + fail("CONTRACT_INVALID", f"case {case_name} status mismatch") + any_unstable = any_unstable or unstable + any_regression = any_regression or regression + any_diagnostic_rejection = any_diagnostic_rejection or diagnostic_rejection + + expected_classification = ( + "BENCHMARK_ENVIRONMENT_UNSTABLE" + if any_unstable + else "PERFORMANCE_REGRESSION" + if any_regression + else "DIAGNOSTIC_REJECTED" + if any_diagnostic_rejection + else "DIAGNOSTIC_QUALIFIED" + if report["mode"] == "diagnostic" + else "PASS" + ) + if ( + report["mode"] == "diagnostic" + and profile_elapsed_ms > DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + and expected_classification == "DIAGNOSTIC_QUALIFIED" + ): + expected_classification = "DIAGNOSTIC_REJECTED" + if report["classification"] != expected_classification: + fail("CONTRACT_INVALID", "paired report classification does not match case evidence") + + try: + distributions = raw_gate.require_exact_fields( + report["operational_counter_distributions"], + {"reference", "candidate"}, + "operational distributions", + ) + for side in ("reference", "candidate"): + side_cases = raw_gate.require_exact_fields( + distributions[side], set(ORDERED_CASES), f"{side} operational distributions" + ) + for case_name in ORDERED_CASES: + counters = raw_gate.require_exact_fields( + side_cases[case_name], + set(raw_gate.OPERATIONAL_COUNTER_FIELDS), + f"{side} {case_name} counters", + ) + for field in raw_gate.OPERATIONAL_COUNTER_FIELDS: + summary = raw_gate.require_exact_fields( + counters[field], {"min", "max", "values"}, f"{side} {case_name} {field}" + ) + if not isinstance(summary["values"], list) or not summary["values"]: + fail("CONTRACT_INVALID", f"{side} {case_name} {field} values are empty") + values = [ + raw_gate.require_nonnegative_integer(value, f"{side} {case_name} {field}") + for value in summary["values"] + ] + if values != sorted(set(values)) or summary["min"] != min(values) or summary["max"] != max(values): + fail("CONTRACT_INVALID", f"{side} {case_name} {field} distribution mismatch") + hard = raw_gate.require_exact_fields( + report["hard_state_comparison"], {"status", "case_count"}, "hard-state comparison" + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if hard != {"status": "equal", "case_count": len(ORDERED_CASES)}: + fail("CORRECTNESS_REGRESSION", "hard-state comparison is incomplete") + try: + cleanup = raw_gate.require_exact_fields( + report["cleanup"], {"status", "attempted", "succeeded", "failed"}, "paired cleanup" + ) + attempted = raw_gate.require_nonnegative_integer(cleanup["attempted"], "cleanup attempted") + succeeded = raw_gate.require_nonnegative_integer(cleanup["succeeded"], "cleanup succeeded") + failed_count = raw_gate.require_nonnegative_integer(cleanup["failed"], "cleanup failed") + except raw_gate.GateError as exc: + fail("EVIDENCE_INTEGRITY_FAILURE", str(exc)) + if cleanup["status"] != "complete" or failed_count != 0 or attempted != succeeded: + fail("EVIDENCE_INTEGRITY_FAILURE", "paired cleanup is incomplete") + expected_cleanup = (2 + report["pair_count"] * 2) * len(ORDERED_CASES) + if attempted != expected_cleanup: + fail("EVIDENCE_INTEGRITY_FAILURE", "paired cleanup inventory count mismatch") + + try: + provenance = raw_gate.require_exact_fields( + report["provenance"], + { + "event_name", + "repository_id", + "runner_os", + "runner_image", + "runner_arch", + "cpu_count", + "go_version", + "postgres_version", + "database_image_digest", + }, + "paired provenance", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if any(provenance[field] in (None, "", "unknown") for field in provenance): + fail("EXECUTION_CONTRACT_MISMATCH", "paired provenance is incomplete") + validate_repository_id(provenance["repository_id"]) + if not re.fullmatch(r"sha256:[0-9a-f]{64}", str(provenance["database_image_digest"])): + fail("EXECUTION_CONTRACT_MISMATCH", "PostgreSQL image digest is invalid") + if isinstance(provenance["cpu_count"], bool) or not isinstance(provenance["cpu_count"], int) or provenance["cpu_count"] <= 0: + fail("EXECUTION_CONTRACT_MISMATCH", "provenance cpu_count is invalid") + raw_gate.validate_no_sensitive_evidence(report, "paired comparison report") + return report + + +def _expected_profile_artifact_files(report: dict[str, Any]) -> set[str]: + expected = {"paired-comparison.json"} + inventory_key = ( + "invocation_inventory" if report["status"] == "complete" else "attempted_invocations" + ) + for index, invocation in enumerate(report[inventory_key], start=1): + expected.add( + require_relative_artifact_path( + invocation["raw_file"], f"artifact invocation {index} raw file" + ).as_posix() + ) + expected.add( + require_relative_artifact_path( + invocation["stderr_file"], f"artifact invocation {index} stderr file" + ).as_posix() + ) + if report["status"] == "failed" and report["active_invocation"] is not None: + active = report["active_invocation"] + if active["raw_capture_present"]: + expected.add( + require_relative_artifact_path(active["raw_file"], "active invocation raw file").as_posix() + ) + if active["stderr_capture_present"]: + expected.add( + require_relative_artifact_path( + active["stderr_file"], "active invocation stderr file" + ).as_posix() + ) + if report["status"] == "complete" and report["mode"] == "production": + expected.update( + { + "governance/reference-manifest.json", + "governance/threshold-policy.json", + } + ) + return expected + + +def _read_artifact_capture(path: pathlib.Path, label: str) -> None: + try: + value = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + fail("EVIDENCE_INTEGRITY_FAILURE", f"read {label}: {exc}") + if "\x00" in value: + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} contains a NUL byte") + if _capture_text_is_sensitive(value): + fail("EVIDENCE_INTEGRITY_FAILURE", f"{label} contains sensitive content") + + +def validate_profile_artifact( + directory: pathlib.Path, *, expected_profile: str, expected_mode: str +) -> dict[str, Any]: + if expected_mode not in DECISION_MODES: + fail("CONTRACT_INVALID", f"unknown decision mode {expected_mode!r}") + validate_checksums(directory) + report_path = _contained_regular_file( + directory, pathlib.PurePosixPath("paired-comparison.json"), "paired report" + ) + try: + report = validate_report_summary( + load_json_strict(report_path), expected_profile=expected_profile + ) + except raw_gate.GateError as exc: + if isinstance(exc, PairedGateError): + raise + fail("CONTRACT_INVALID", str(exc)) + if report["mode"] != expected_mode: + fail( + "REFERENCE_GOVERNANCE_INVALID", + f"{expected_mode} decision cannot consume {report['mode']} artifacts", + ) + validate_checksums( + directory, expected_files=_expected_profile_artifact_files(report) + ) + if report["status"] == "failed": + for index, invocation in enumerate(report["attempted_invocations"], start=1): + for field in ("raw_file", "stderr_file"): + relative = require_relative_artifact_path( + invocation[field], f"failure invocation {index} {field}" + ) + path = _contained_regular_file( + directory, relative, f"failure invocation {index} {field}" + ) + _read_artifact_capture(path, f"failure invocation {index} {field}") + active = report["active_invocation"] + if active is not None: + for field, presence_field in ( + ("raw_file", "raw_capture_present"), + ("stderr_file", "stderr_capture_present"), + ): + if active[presence_field]: + relative = require_relative_artifact_path( + active[field], f"active failure invocation {field}" + ) + path = _contained_regular_file( + directory, relative, f"active failure invocation {field}" + ) + _read_artifact_capture(path, f"active failure invocation {field}") + return report + thresholds: dict[str, float] | None = None + if expected_mode == "production": + if not PRODUCTION_SAMPLING_AUTHORIZED: + fail( + "REFERENCE_GOVERNANCE_INVALID", + "production paired decisions are not authorized in this repository state", + ) + + repository = _repository_root() + repository_manifest = _governed_repository_file( + repository, GOVERNED_MANIFEST_RELATIVE, "governed reference manifest" + ) + repository_thresholds = _governed_repository_file( + repository, GOVERNED_THRESHOLD_RELATIVE, "governed threshold policy" + ) + artifact_manifest = _contained_regular_file( + directory, + pathlib.PurePosixPath("governance/reference-manifest.json"), + "artifact reference manifest", + ) + artifact_thresholds = _contained_regular_file( + directory, + pathlib.PurePosixPath("governance/threshold-policy.json"), + "artifact threshold policy", + ) + if ( + _binary_hash(artifact_manifest) != _binary_hash(repository_manifest) + or _binary_hash(artifact_thresholds) != _binary_hash(repository_thresholds) + ): + fail("REFERENCE_GOVERNANCE_INVALID", "artifact governance differs from repository") + manifest = load_json_strict(artifact_manifest) + policy = load_json_strict(artifact_thresholds) + validate_reference_manifest(manifest) + thresholds = validate_threshold_policy(policy) + identity = report["identity"] + verify_reference_governance( + manifest, + reference_sha=identity["reference_sha"], + candidate_sha=identity["candidate_sha"], + repository=repository, + ) + governance = report["governance"] + if ( + governance["manifest_sha256"] != _binary_hash(artifact_manifest) + or governance["threshold_policy_id"] != policy["policy_id"] + or governance["threshold_policy_sha256"] != _binary_hash(artifact_thresholds) + or manifest["threshold_policy_id"] != policy["policy_id"] + or manifest["threshold_policy_sha256"] != _binary_hash(artifact_thresholds) + ): + fail("REFERENCE_GOVERNANCE_INVALID", "artifact governance identity mismatch") + + warmups: list[dict[str, Any]] = [] + measured: list[dict[str, Any]] = [] + for invocation in report["invocation_inventory"]: + raw_relative = require_relative_artifact_path( + invocation["raw_file"], "artifact raw report" + ) + stderr_relative = require_relative_artifact_path( + invocation["stderr_file"], "artifact stderr" + ) + raw_path = _contained_regular_file(directory, raw_relative, "artifact raw report") + stderr_path = _contained_regular_file(directory, stderr_relative, "artifact stderr") + try: + envelope = load_json_strict(raw_path) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + _read_artifact_capture(stderr_path, "artifact stderr") + record = { + "kind": invocation["kind"], + "pair_ordinal": invocation["pair_ordinal"], + "position": invocation["position"], + "side": invocation["side"], + "envelope": envelope, + } + (warmups if invocation["kind"] == "warmup" else measured).append(record) + + profile = report["profile"] + recomputed = compare_records( + measured, + pair_count=report["pair_count"], + dataset=profile["dataset"], + workers=profile["workers"], + compression=profile["compression"], + mode=expected_mode, + thresholds=thresholds, + ) + validate_warmups( + warmups, + measured, + dataset=profile["dataset"], + workers=profile["workers"], + compression=profile["compression"], + ) + if ( + expected_mode == "diagnostic" + and report["profile_elapsed_ms"] > DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + and recomputed["classification"] == "DIAGNOSTIC_QUALIFIED" + ): + recomputed["classification"] = "DIAGNOSTIC_REJECTED" + for field in ( + "classification", + "fixture", + "cases", + "operational_counter_distributions", + "hard_state_comparison", + ): + if report[field] != recomputed[field]: + fail("EVIDENCE_INTEGRITY_FAILURE", f"paired report {field} differs from raw evidence") + return report + + +def decision_classification(classifications: list[str], *, mode: str) -> str: + if mode not in DECISION_MODES: + fail("CONTRACT_INVALID", f"unknown decision mode {mode!r}") + if mode == "diagnostic": + if any(value in {"PASS", "PERFORMANCE_REGRESSION"} for value in classifications): + fail("CONTRACT_INVALID", "diagnostic decision contains a production classification") + if classifications and all( + value == "DIAGNOSTIC_QUALIFIED" for value in classifications + ): + return "DIAGNOSTIC_QUALIFIED" + elif any(value in {"DIAGNOSTIC_QUALIFIED", "DIAGNOSTIC_REJECTED"} for value in classifications): + fail("REFERENCE_GOVERNANCE_INVALID", "production decision contains diagnostic evidence") + for classification in DECISION_PRECEDENCE: + if classification in classifications: + return classification + fail("CONTRACT_INVALID", "decision contains no recognized classification") + raise AssertionError("unreachable") + + +DECISION_FIELDS = { + "schema_version", + "evidence_policy_version", + "report_kind", + "status", + "mode", + "classification", + "contract_version", + "decision_scope", + "authority", + "production_authority", + "identity", + "profiles", + "matrix_coverage", + "evidence", +} + +DECISION_FAILURE_FIELDS = DECISION_FIELDS - {"evidence"} | {"evidence", "failure_reason"} + + +def _validate_decision_failure_report( + report: Any, *, expected_mode: str | None = None +) -> dict[str, Any]: + try: + report = raw_gate.require_exact_fields( + report, DECISION_FAILURE_FIELDS, "paired failure decision" + ) + identity = raw_gate.require_exact_fields( + report["identity"], + { + "reference_sha", + "candidate_sha", + "reference_binary_sha256", + "candidate_binary_sha256", + }, + "failure decision identity", + ) + evidence = raw_gate.require_exact_fields( + report["evidence"], + { + "checksum_verification", + "raw_reconstruction", + "pair_inventory", + "hard_state", + "counters", + "cleanup", + }, + "failure decision evidence", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + mode = report["mode"] + if ( + report["schema_version"] != SCHEMA_VERSION + or report["evidence_policy_version"] != EVIDENCE_POLICY_VERSION + or report["report_kind"] != DECISION_KIND + or report["status"] != "failed" + or report["contract_version"] != CONTRACT_VERSION + or report["classification"] not in CLASSIFICATIONS - SUCCESS_CLASSIFICATIONS + or mode not in DECISION_MODES + ): + fail("CONTRACT_INVALID", "failure decision identity mismatch") + if expected_mode is not None and mode != expected_mode: + fail("REFERENCE_GOVERNANCE_INVALID", "failure decision mode mismatch") + expected_authority = authority_contract(mode) + if any(report[field] != expected_authority[field] for field in expected_authority): + fail("REFERENCE_GOVERNANCE_INVALID", "failure decision authority mismatch") + if not isinstance(report["failure_reason"], str) or not report["failure_reason"]: + fail("CONTRACT_INVALID", "failure decision reason is missing") + for field, length in ( + ("reference_sha", 40), + ("candidate_sha", 40), + ("reference_binary_sha256", 64), + ("candidate_binary_sha256", 64), + ): + value = identity[field] + if value is not None and ( + not isinstance(value, str) or re.fullmatch(rf"[0-9a-f]{{{length}}}", value) is None + ): + fail("CONTRACT_INVALID", f"failure decision {field} is invalid") + profiles = report["profiles"] + if ( + not isinstance(profiles, list) + or any(not isinstance(item, dict) for item in profiles) + or [item.get("profile") for item in profiles] != sorted(PROFILE_MATRIX) + ): + fail("PAIR_INVENTORY_INVALID", "failure decision profile inventory mismatch") + for item in profiles: + try: + item = raw_gate.require_exact_fields( + item, + { + "profile", + "status", + "classification", + "checksum_verification", + "raw_reconstruction", + }, + "failure decision profile", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if item["status"] not in {"verified", "failed", "invalid", "missing", "not_evaluated"}: + fail("CONTRACT_INVALID", "failure decision profile status is invalid") + if item["checksum_verification"] not in {"verified", "failed", "missing", "not_evaluated"}: + fail("CONTRACT_INVALID", "failure decision checksum state is invalid") + if item["raw_reconstruction"] not in {"verified", "failed", "not_evaluated"}: + fail("CONTRACT_INVALID", "failure decision raw state is invalid") + if item["classification"] is not None and item["classification"] not in CLASSIFICATIONS: + fail("CONTRACT_INVALID", "failure decision profile classification is invalid") + if item["status"] == "verified" and ( + item["checksum_verification"] != "verified" + or item["raw_reconstruction"] != "verified" + ): + fail("EVIDENCE_INTEGRITY_FAILURE", "verified failure-decision profile lacks evidence") + if item["status"] in {"missing", "not_evaluated"} and ( + item["checksum_verification"] == "verified" + or item["raw_reconstruction"] == "verified" + ): + fail("EVIDENCE_INTEGRITY_FAILURE", "unavailable failure-decision profile is verified") + if report["matrix_coverage"] != sorted(PROFILE_MATRIX): + fail("PAIR_INVENTORY_INVALID", "failure decision matrix coverage mismatch") + try: + checksum_evidence = raw_gate.require_exact_fields( + evidence["checksum_verification"], + {"status", "verified_profile_count"}, + "failure decision checksum evidence", + ) + raw_evidence = raw_gate.require_exact_fields( + evidence["raw_reconstruction"], + {"status", "verified_profile_count"}, + "failure decision raw evidence", + ) + for field in ("pair_inventory", "hard_state", "counters", "cleanup"): + raw_gate.require_exact_fields( + evidence[field], {"status"}, f"failure decision {field} evidence" + ) + checksum_count = raw_gate.require_nonnegative_integer( + checksum_evidence["verified_profile_count"], "verified checksum profile count" + ) + raw_count = raw_gate.require_nonnegative_integer( + raw_evidence["verified_profile_count"], "verified raw profile count" + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if ( + checksum_evidence["status"] not in {"failed", "incomplete"} + or raw_evidence["status"] not in {"failed", "incomplete"} + or checksum_count != sum(item["checksum_verification"] == "verified" for item in profiles) + or raw_count != sum(item["raw_reconstruction"] == "verified" for item in profiles) + or evidence["pair_inventory"]["status"] != "incomplete" + or evidence["hard_state"]["status"] != "not_verified" + or evidence["counters"]["status"] != "not_verified" + or evidence["cleanup"]["status"] != "not_verified" + ): + fail("EVIDENCE_INTEGRITY_FAILURE", "failure decision evidence overclaims verification") + raw_gate.validate_no_sensitive_evidence(report, "paired failure decision") + return report + + +def validate_decision_report(report: Any, *, expected_mode: str | None = None) -> dict[str, Any]: + if isinstance(report, dict) and report.get("status") == "failed": + return _validate_decision_failure_report(report, expected_mode=expected_mode) + try: + report = raw_gate.require_exact_fields(report, DECISION_FIELDS, "paired decision report") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + mode = report["mode"] + if expected_mode is not None and mode != expected_mode: + fail("REFERENCE_GOVERNANCE_INVALID", "decision report mode mismatch") + if mode not in DECISION_MODES: + fail("CONTRACT_INVALID", "paired decision mode is invalid") + expected_authority = authority_contract(mode) + if any(report[field] != expected_authority[field] for field in expected_authority): + fail("REFERENCE_GOVERNANCE_INVALID", "paired decision authority mismatch") + if ( + report["schema_version"] != SCHEMA_VERSION + or report["evidence_policy_version"] != EVIDENCE_POLICY_VERSION + or report["report_kind"] != DECISION_KIND + or report["status"] != "complete" + or report["contract_version"] != CONTRACT_VERSION + ): + fail("CONTRACT_INVALID", "paired decision identity mismatch") + try: + identity = raw_gate.require_exact_fields( + report["identity"], + { + "reference_sha", + "candidate_sha", + "reference_binary_sha256", + "candidate_binary_sha256", + }, + "paired decision identity", + ) + require_sha(identity["reference_sha"], "decision reference SHA") + require_sha(identity["candidate_sha"], "decision candidate SHA") + raw_gate.require_sha256(identity["reference_binary_sha256"], "decision reference binary") + raw_gate.require_sha256(identity["candidate_binary_sha256"], "decision candidate binary") + except raw_gate.GateError as exc: + if isinstance(exc, PairedGateError): + raise + fail("CONTRACT_INVALID", str(exc)) + if mode == "diagnostic" and identity["reference_binary_sha256"] != identity[ + "candidate_binary_sha256" + ]: + fail("BINARY_IDENTITY_INVALID", "diagnostic decision binaries are not byte-identical") + + profiles = report["profiles"] + if not isinstance(profiles, list) or len(profiles) != len(PROFILE_MATRIX): + fail("CONTRACT_INVALID", "paired decision profile inventory is incomplete") + classifications = [] + for index, summary in enumerate(profiles, start=1): + try: + summary = raw_gate.require_exact_fields( + summary, + { + "profile", + "classification", + "profile_elapsed_ms", + "pair_count", + "warmup_order", + "measured_order", + "checksum_verification", + "raw_reconstruction", + "cases", + "hard_state_comparison", + "counter_validation", + "cleanup", + }, + f"decision profile {index}", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + expected_name = sorted(PROFILE_MATRIX)[index - 1] + if summary["profile"] != expected_name: + fail("EXECUTION_CONTRACT_MISMATCH", "decision profile order or identity mismatch") + if summary["classification"] not in CLASSIFICATIONS: + fail("CONTRACT_INVALID", "decision profile classification is invalid") + classifications.append(summary["classification"]) + try: + raw_gate.require_number( + summary["profile_elapsed_ms"], f"{expected_name} elapsed duration", positive=True + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + expected_pairs = 10 if mode == "diagnostic" else 5 + if ( + summary["pair_count"] != expected_pairs + or summary["warmup_order"] != list(WARMUP_ORDER) + or summary["measured_order"] + != [list(pair) for pair in measured_order(expected_pairs)] + or summary["checksum_verification"] != "verified" + or summary["raw_reconstruction"] != "verified" + ): + fail("PAIR_INVENTORY_INVALID", f"{expected_name} decision inventory mismatch") + cases = summary["cases"] + if not isinstance(cases, list) or len(cases) != len(PERFORMANCE_CASES): + fail("CONTRACT_INVALID", f"{expected_name} qualification case inventory mismatch") + profile_unstable = False + profile_rejected = False + for case_index, case in enumerate(cases): + try: + case = raw_gate.require_exact_fields( + case, + {"case", "median_ratio", "paired_mad_ratio_pct", "status"}, + f"{expected_name} decision case {case_index + 1}", + ) + raw_gate.require_number(case["median_ratio"], "decision median ratio", positive=True) + raw_gate.require_number(case["paired_mad_ratio_pct"], "decision paired MAD") + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if case["case"] != PERFORMANCE_CASES[case_index]: + fail("CONTRACT_INVALID", "decision qualification case order mismatch") + if mode == "diagnostic": + unstable = Decimal(str(case["paired_mad_ratio_pct"])) > Decimal("2.5") + rejected = not Decimal("0.95") <= Decimal( + str(case["median_ratio"]) + ) <= Decimal("1.05") + expected_status = ( + "unstable" if unstable else "qualification_rejected" if rejected else "pass" + ) + if case["status"] != expected_status: + fail("CONTRACT_INVALID", "decision qualification case status mismatch") + profile_unstable = profile_unstable or unstable + profile_rejected = profile_rejected or rejected + try: + hard = raw_gate.require_exact_fields( + summary["hard_state_comparison"], {"status", "case_count"}, "decision hard state" + ) + counters = raw_gate.require_exact_fields( + summary["counter_validation"], {"status", "case_count"}, "decision counters" + ) + cleanup = raw_gate.require_exact_fields( + summary["cleanup"], {"status", "attempted", "succeeded", "failed"}, "decision cleanup" + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + if hard != {"status": "equal", "case_count": len(ORDERED_CASES)}: + fail("CORRECTNESS_REGRESSION", "decision hard-state evidence is incomplete") + if counters != {"status": "valid", "case_count": len(ORDERED_CASES)}: + fail("EVIDENCE_INTEGRITY_FAILURE", "decision counter evidence is incomplete") + expected_cleanup = (2 + expected_pairs * 2) * len(ORDERED_CASES) + if cleanup != { + "status": "complete", + "attempted": expected_cleanup, + "succeeded": expected_cleanup, + "failed": 0, + }: + fail("EVIDENCE_INTEGRITY_FAILURE", "decision cleanup evidence is incomplete") + if mode == "diagnostic": + expected_profile_classification = ( + "BENCHMARK_ENVIRONMENT_UNSTABLE" + if profile_unstable + else "DIAGNOSTIC_REJECTED" + if profile_rejected + or summary["profile_elapsed_ms"] > DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + else "DIAGNOSTIC_QUALIFIED" + ) + if summary["classification"] != expected_profile_classification: + fail("CONTRACT_INVALID", "decision profile classification mismatch") + if report["matrix_coverage"] != sorted(PROFILE_MATRIX): + fail("CONTRACT_INVALID", "paired decision matrix coverage mismatch") + + try: + evidence = raw_gate.require_exact_fields( + report["evidence"], + { + "checksum_verification", + "raw_reconstruction", + "pair_inventory", + "qualification_bounds", + "hard_state", + "counters", + "cleanup", + }, + "paired decision evidence", + ) + except raw_gate.GateError as exc: + fail("CONTRACT_INVALID", str(exc)) + expected_pairs = 10 if mode == "diagnostic" else 5 + expected_evidence = { + "checksum_verification": {"status": "verified", "profile_count": 4}, + "raw_reconstruction": { + "status": "verified", + "profile_count": 4, + "raw_schema_version": RAW_SCHEMA_VERSION, + "diagnostic_schema_version": DIAGNOSTIC_SCHEMA_VERSION, + }, + "pair_inventory": { + "status": "complete", + "warmup_order": list(WARMUP_ORDER), + "pair_count": expected_pairs, + "measured_order": [list(pair) for pair in measured_order(expected_pairs)], + }, + "qualification_bounds": { + "median_ratio_minimum": 0.95 if mode == "diagnostic" else None, + "median_ratio_maximum": 1.05 if mode == "diagnostic" else None, + "paired_mad_ratio_maximum_pct": 2.5 if mode == "diagnostic" else None, + "profile_elapsed_maximum_ms": ( + DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS if mode == "diagnostic" else None + ), + }, + "hard_state": {"status": "equal", "profile_count": 4}, + "counters": {"status": "valid", "profile_count": 4}, + "cleanup": {"status": "complete", "profile_count": 4}, + } + if evidence != expected_evidence: + fail("EVIDENCE_INTEGRITY_FAILURE", "paired decision evidence summary mismatch") + expected_classification = decision_classification(classifications, mode=mode) + if report["classification"] != expected_classification: + fail("CONTRACT_INVALID", "paired decision classification mismatch") + raw_gate.validate_no_sensitive_evidence(report, "paired decision report") + return report + + +def validate_decision_artifact(directory: pathlib.Path, *, expected_mode: str) -> dict[str, Any]: + validate_checksums(directory, expected_files={"paired-decision.json"}) + path = _contained_regular_file( + directory, pathlib.PurePosixPath("paired-decision.json"), "paired decision report" + ) + return validate_decision_report(load_json_strict(path), expected_mode=expected_mode) + + +def decision_command(args: argparse.Namespace) -> int: + _create_output_directory(args.output_dir) + args._output_owned = True + args._decision_state = { + "profiles": { + name: { + "profile": name, + "status": "not_evaluated", + "classification": None, + "checksum_verification": "not_evaluated", + "raw_reconstruction": "not_evaluated", + } + for name in sorted(PROFILE_MATRIX) + }, + "identity": { + "reference_sha": None, + "candidate_sha": None, + "reference_binary_sha256": None, + "candidate_binary_sha256": None, + }, + } + if args.mode not in DECISION_MODES: + fail("CONTRACT_INVALID", f"unknown decision mode {args.mode!r}") + if args.mode == "production" and not PRODUCTION_SAMPLING_AUTHORIZED: + fail( + "REFERENCE_GOVERNANCE_INVALID", + "production paired decisions are not authorized in this repository state", + ) + profile_args: dict[str, pathlib.Path] = {} + for value in args.profile: + if "=" not in value: + fail("CONTRACT_INVALID", "decision profile must be NAME=ARTIFACT_DIRECTORY") + name, path = value.split("=", 1) + if name in profile_args: + fail("CONTRACT_INVALID", f"duplicate decision profile {name!r}") + profile_args[name] = pathlib.Path(path) + if set(profile_args) != set(PROFILE_MATRIX): + fail("CONTRACT_INVALID", "decision requires exactly the four paired profiles") + + summaries: list[dict[str, Any]] = [] + common_source: tuple[Any, ...] | None = None + common_hashes: tuple[Any, ...] | None = None + common_provenance: tuple[Any, ...] | None = None + common_governance: tuple[Any, ...] | None = None + common_inventory: tuple[Any, ...] | None = None + for name in sorted(profile_args): + directory = profile_args[name] + state = args._decision_state["profiles"][name] + state["status"] = "missing" if not directory.is_dir() else "invalid" + state["checksum_verification"] = ( + "missing" if not (directory / "checksums.sha256").is_file() else "failed" + ) + state["raw_reconstruction"] = "not_evaluated" + try: + report = validate_profile_artifact( + directory, expected_profile=name, expected_mode=args.mode + ) + except PairedGateError as exc: + state["classification"] = exc.classification + raise + state["checksum_verification"] = "verified" + state["raw_reconstruction"] = "verified" + if report["status"] != "complete": + state["status"] = "failed" + state["classification"] = report["classification"] + fail(report["classification"], f"profile {name} did not complete") + state["status"] = "verified" + state["classification"] = report["classification"] + identity = report["identity"] + source_key = ( + identity["reference_sha"], + identity["candidate_sha"], + report["contract_version"], + ) + hash_key = ( + identity["reference_binary_sha256"], + identity["candidate_binary_sha256"], + ) + provenance = report["provenance"] + provenance_key = ( + provenance["repository_id"], + provenance["go_version"], + provenance["postgres_version"], + provenance["database_image_digest"], + ) + governance = report.get("governance") + governance_key = ( + governance["manifest_sha256"], + governance["threshold_policy_id"], + governance["threshold_policy_sha256"], + ) if governance is not None else None + inventory_key = ( + report["pair_count"], + tuple(report["warmup_order"]), + tuple(tuple(pair) for pair in report["measured_order"]), + ) + if common_source is None: + common_source = source_key + common_hashes = hash_key + common_provenance = provenance_key + common_governance = governance_key + common_inventory = inventory_key + args._decision_state["identity"] = { + "reference_sha": identity["reference_sha"], + "candidate_sha": identity["candidate_sha"], + "reference_binary_sha256": identity["reference_binary_sha256"], + "candidate_binary_sha256": identity["candidate_binary_sha256"], + } + elif source_key != common_source: + fail("EXECUTION_CONTRACT_MISMATCH", "profile source or contract identities differ") + elif hash_key != common_hashes: + fail("BINARY_IDENTITY_INVALID", "profile binary identities differ") + elif provenance_key != common_provenance or governance_key != common_governance: + fail("EXECUTION_CONTRACT_MISMATCH", "profile source or environment pins differ") + elif inventory_key != common_inventory: + fail("PAIR_INVENTORY_INVALID", "profile pair inventories differ") + summaries.append( + { + "profile": name, + "classification": report["classification"], + "profile_elapsed_ms": report["profile_elapsed_ms"], + "pair_count": report["pair_count"], + "warmup_order": report["warmup_order"], + "measured_order": report["measured_order"], + "checksum_verification": "verified", + "raw_reconstruction": "verified", + "cases": [ + { + "case": case["case"], + "median_ratio": case["median_ratio"], + "paired_mad_ratio_pct": case["paired_mad_ratio_pct"], + "status": case["status"], + } + for case in report["cases"] + if case["performance_gated"] + ], + "hard_state_comparison": report["hard_state_comparison"], + "counter_validation": {"status": "valid", "case_count": len(ORDERED_CASES)}, + "cleanup": report["cleanup"], + } + ) + + classification = decision_classification( + [item["classification"] for item in summaries], mode=args.mode + ) + authority = authority_contract(args.mode) + pair_count = 10 if args.mode == "diagnostic" else 5 + decision = { + "schema_version": 1, + "evidence_policy_version": 2, + "report_kind": DECISION_KIND, + "status": "complete", + "mode": args.mode, + "classification": classification, + "contract_version": CONTRACT_VERSION, + **authority, + "identity": { + "reference_sha": common_source[0], + "candidate_sha": common_source[1], + "reference_binary_sha256": common_hashes[0], + "candidate_binary_sha256": common_hashes[1], + }, + "profiles": summaries, + "matrix_coverage": sorted(PROFILE_MATRIX), + "evidence": { + "checksum_verification": {"status": "verified", "profile_count": 4}, + "raw_reconstruction": { + "status": "verified", + "profile_count": 4, + "raw_schema_version": RAW_SCHEMA_VERSION, + "diagnostic_schema_version": DIAGNOSTIC_SCHEMA_VERSION, + }, + "pair_inventory": { + "status": "complete", + "warmup_order": list(WARMUP_ORDER), + "pair_count": pair_count, + "measured_order": [list(pair) for pair in measured_order(pair_count)], + }, + "qualification_bounds": { + "median_ratio_minimum": 0.95 if args.mode == "diagnostic" else None, + "median_ratio_maximum": 1.05 if args.mode == "diagnostic" else None, + "paired_mad_ratio_maximum_pct": 2.5 if args.mode == "diagnostic" else None, + "profile_elapsed_maximum_ms": ( + DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + if args.mode == "diagnostic" + else None + ), + }, + "hard_state": {"status": "equal", "profile_count": 4}, + "counters": {"status": "valid", "profile_count": 4}, + "cleanup": {"status": "complete", "profile_count": 4}, + }, + } + validate_decision_report(decision, expected_mode=args.mode) + raw_gate.write_json(args.output_dir / "paired-decision.json", decision) + _write_checksums(args.output_dir) + print(json.dumps({"classification": classification, "report": "paired-decision.json"})) + return 0 if classification in SUCCESS_CLASSIFICATIONS else 1 + + +def _write_decision_failure_artifact(args: argparse.Namespace, exc: PairedGateError) -> str: + output_dir = args.output_dir + if output_dir.is_symlink() or not output_dir.is_dir(): + raise PairedGateError( + "EVIDENCE_INTEGRITY_FAILURE", "owned decision artifact directory is unavailable" + ) + state = getattr(args, "_decision_state", {}) + profiles_by_name = state.get("profiles", {}) + profiles = [] + for name in sorted(PROFILE_MATRIX): + profiles.append( + profiles_by_name.get( + name, + { + "profile": name, + "status": "not_evaluated", + "classification": None, + "checksum_verification": "not_evaluated", + "raw_reconstruction": "not_evaluated", + }, + ) + ) + checksum_verified = sum( + item["checksum_verification"] == "verified" for item in profiles + ) + raw_verified = sum(item["raw_reconstruction"] == "verified" for item in profiles) + authority = authority_contract(args.mode) + decision = { + "schema_version": SCHEMA_VERSION, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": DECISION_KIND, + "status": "failed", + "mode": args.mode, + "classification": exc.classification, + "contract_version": CONTRACT_VERSION, + **authority, + "identity": state.get( + "identity", + { + "reference_sha": None, + "candidate_sha": None, + "reference_binary_sha256": None, + "candidate_binary_sha256": None, + }, + ), + "profiles": profiles, + "matrix_coverage": sorted(PROFILE_MATRIX), + "failure_reason": exc.classification.lower(), + "evidence": { + "checksum_verification": { + "status": "failed" if any( + item["checksum_verification"] in {"failed", "missing"} + for item in profiles + ) else "incomplete", + "verified_profile_count": checksum_verified, + }, + "raw_reconstruction": { + "status": "failed" if any( + item["raw_reconstruction"] == "failed" for item in profiles + ) else "incomplete", + "verified_profile_count": raw_verified, + }, + "pair_inventory": {"status": "incomplete"}, + "hard_state": {"status": "not_verified"}, + "counters": {"status": "not_verified"}, + "cleanup": {"status": "not_verified"}, + }, + } + validate_decision_report(decision, expected_mode=args.mode) + raw_gate.write_json(output_dir / "paired-decision.json", decision) + _write_checksums(output_dir) + return exc.classification + + +def _write_failure_artifact(args: argparse.Namespace, exc: PairedGateError) -> str: + """Write a sanitized immutable summary of only the validated prefix.""" + output_dir = args.output_dir + if output_dir.is_symlink() or not output_dir.is_dir(): + raise PairedGateError( + "EVIDENCE_INTEGRITY_FAILURE", "owned failure artifact directory is unavailable" + ) + + def available_hash(path: pathlib.Path) -> str | None: + try: + return _binary_hash(path) if path.is_file() else None + except OSError: + return None + + _sanitize_failure_captures(output_dir) + attempted_invocations = [] + validated_records: list[tuple[str, list[dict[str, Any]]]] = [] + raw_root = output_dir / "raw" + if raw_root.is_dir(): + for path in sorted(raw_root.rglob("*.json")): + relative = path.relative_to(output_dir).as_posix() + warmup_match = re.fullmatch(r"raw/warmup-(\d{2})-(reference|candidate)\.json", relative) + pair_match = re.fullmatch( + r"raw/pair-(\d{2})/(\d{2})-(reference|candidate)\.json", relative + ) + invocation: dict[str, Any] | None = None + if warmup_match: + position, side = warmup_match.groups() + invocation = { + "kind": "warmup", + "pair_ordinal": None, + "position": int(position), + "side": side, + "raw_file": relative, + "stderr_file": path.with_suffix(".stderr").relative_to(output_dir).as_posix(), + } + elif pair_match: + ordinal, position, side = pair_match.groups() + invocation = { + "kind": "measured", + "pair_ordinal": int(ordinal), + "position": int(position), + "side": side, + "raw_file": relative, + "stderr_file": path.with_suffix(".stderr").relative_to(output_dir).as_posix(), + } + if invocation is None or not path.with_suffix(".stderr").is_file(): + continue + try: + _, rows = validate_raw_report( + load_json_strict(path), + dataset=args.dataset, + workers=args.workers, + compression=args.compression, + ) + except (PairedGateError, raw_gate.GateError): + continue + attempted_invocations.append(invocation) + validated_records.append((invocation["side"], rows)) + attempted_invocations.sort( + key=lambda item: ( + 0 if item["kind"] == "warmup" else 1, + item["pair_ordinal"] or 0, + item["position"], + ) + ) + + state = getattr(args, "_profile_state", {}) + cleanup_result = _cleanup_interrupted_profile(args) + classification = ( + "EVIDENCE_INTEGRITY_FAILURE" + if cleanup_result["status"] != "complete" + else exc.classification + ) + active = state.get("active_invocation") + active_report = None + if active is not None: + if active["kind"] == "warmup": + active_raw = pathlib.Path("raw") / f"warmup-{active['position']:02d}-{active['side']}.json" + else: + active_raw = ( + pathlib.Path("raw") + / f"pair-{active['pair_ordinal']:02d}" + / f"{active['position']:02d}-{active['side']}.json" + ) + active_stderr = active_raw.with_suffix(".stderr") + active_report = { + **active, + "raw_file": active_raw.as_posix(), + "stderr_file": active_stderr.as_posix(), + "raw_capture_present": (output_dir / active_raw).is_file(), + "stderr_capture_present": (output_dir / active_stderr).is_file(), + "capture_validation": "unvalidated", + "status": "incomplete", + } + + hard_state = "not_evaluated" + if {side for side, _ in validated_records} == {"reference", "candidate"}: + hard_state = "equal" + for case_index in range(len(ORDERED_CASES)): + contracts = { + side: [_semantic_contract(rows[case_index]) for record_side, rows in validated_records if record_side == side] + for side in ("reference", "candidate") + } + if any(values and any(value != values[0] for value in values[1:]) for values in contracts.values()): + hard_state = "mismatch" + break + if all(contracts.values()) and contracts["reference"][0] != contracts["candidate"][0]: + hard_state = "mismatch" + break + + started = state.get("started") + elapsed_ms = max((time.monotonic() - started) * 1000.0, 0.001) if started is not None else 0.001 + cancellation_reason = state.get("cancellation_reason") or classification.lower() + cleanup_active = ( + "not_applicable" + if active_report is None + else "cleaned" + if cleanup_result["status"] == "complete" + else "incomplete" + ) + report = { + "schema_version": SCHEMA_VERSION, + "evidence_policy_version": EVIDENCE_POLICY_VERSION, + "report_kind": REPORT_KIND, + "status": "failed", + "mode": args.mode, + "classification": classification, + "contract_version": CONTRACT_VERSION, + "authority": authority_contract(args.mode), + "identity": { + "reference_sha": args.reference_sha, + "candidate_sha": args.candidate_sha, + "reference_binary_sha256": available_hash(args.reference_binary), + "candidate_binary_sha256": available_hash(args.candidate_binary), + }, + "governance_status": ( + "provisional-diagnostic" if args.mode == "diagnostic" else "not-established" + ), + "profile": { + "codec": "aes-gcm", + "compression": args.compression, + "dataset": args.dataset, + "workers": args.workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "requested_pair_count": args.pairs, + "warmup_order": list(WARMUP_ORDER), + "measured_order": [list(pair) for pair in measured_order(args.pairs)], + "attempted_invocations": attempted_invocations, + "active_invocation": active_report, + "profile_elapsed_ms": elapsed_ms, + "cancellation": { + "reason": cancellation_reason, + "authoritative": False, + }, + "prefix_validation": { + "status": "validated" if attempted_invocations else "not_evaluated", + "raw_report_count": len(attempted_invocations), + "case_row_count": len(attempted_invocations) * len(ORDERED_CASES), + "counter_validation": "valid" if attempted_invocations else "not_evaluated", + "hard_state": hard_state, + }, + "cleanup": { + "status": cleanup_result["status"], + "observed_invocations": len(attempted_invocations), + "required_invocations": 2 + args.pairs * 2, + "completed_cases": len(attempted_invocations) * len(ORDERED_CASES), + "active_invocation": cleanup_active, + "filesystem_entries_removed": cleanup_result["filesystem_entries_removed"], + "databases_removed": cleanup_result["databases_removed"], + "errors": cleanup_result["errors"], + }, + "provenance": _provenance(args), + } + _validate_failure_report(report, expected_profile=None) + raw_gate.write_json(output_dir / "paired-comparison.json", report) + _write_checksums(output_dir) + return classification + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + sample = subparsers.add_parser("sample", help="run one paired benchmark profile") + sample.add_argument("--reference-binary", type=pathlib.Path, required=True) + sample.add_argument("--candidate-binary", type=pathlib.Path, required=True) + sample.add_argument("--reference-sha") + sample.add_argument("--candidate-sha", required=True) + sample.add_argument("--output-dir", type=pathlib.Path, required=True) + sample.add_argument("--dataset", choices=sorted(FIXTURES), required=True) + sample.add_argument("--compression", choices=("none", "zstd"), required=True) + sample.add_argument("--workers", type=int, choices=(1, 4), required=True) + sample.add_argument("--mode", choices=("diagnostic", "production"), required=True) + sample.add_argument("--pairs", type=int, choices=(5, 10), required=True) + sample.add_argument("--command-timeout-seconds", type=int, default=600) + sample.add_argument("--go-version", required=True) + sample.add_argument("--postgres-version", required=True) + sample.add_argument("--database-image-digest", required=True) + sample.set_defaults(handler=sample_command) + + decision = subparsers.add_parser("decision", help="combine four immutable profile artifacts") + decision.add_argument("--mode", choices=DECISION_MODES, required=True) + decision.add_argument("--profile", action="append", required=True) + decision.add_argument("--output-dir", type=pathlib.Path, required=True) + decision.set_defaults(handler=decision_command) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + output_owned = False + previous_sigterm = signal.getsignal(signal.SIGTERM) + + def handle_sigterm(_signum: int, _frame: Any) -> None: + if hasattr(args, "_profile_state"): + args._profile_state["cancellation_reason"] = "external SIGTERM" + raise PairedGateError( + "CI_INFRASTRUCTURE_TIMEOUT", "command terminated before fixed evidence completed" + ) + + try: + if args.command == "sample": + _create_output_directory(args.output_dir) + output_owned = True + signal.signal(signal.SIGTERM, handle_sigterm) + return args.handler(args) + except PairedGateError as exc: + classification = exc.classification + if args.command == "sample" and output_owned: + classification = _write_failure_artifact(args, exc) + elif args.command == "decision" and getattr(args, "_output_owned", False): + classification = _write_decision_failure_artifact(args, exc) + print(json.dumps({"classification": classification, "error": str(exc)}), file=sys.stderr) + return 2 + except raw_gate.GateError as exc: + paired_exc = PairedGateError("CONTRACT_INVALID", str(exc)) + if args.command == "sample" and output_owned: + _write_failure_artifact(args, paired_exc) + elif args.command == "decision" and getattr(args, "_output_owned", False): + _write_decision_failure_artifact(args, paired_exc) + print(json.dumps({"classification": "CONTRACT_INVALID", "error": str(exc)}), file=sys.stderr) + return 2 + except KeyboardInterrupt: + paired_exc = PairedGateError( + "CI_INFRASTRUCTURE_TIMEOUT", "command interrupted before fixed evidence completed" + ) + if args.command == "sample" and output_owned: + _write_failure_artifact(args, paired_exc) + elif args.command == "decision" and getattr(args, "_output_owned", False): + _write_decision_failure_artifact(args, paired_exc) + print( + json.dumps({"classification": paired_exc.classification, "error": str(paired_exc)}), + file=sys.stderr, + ) + return 2 + finally: + signal.signal(signal.SIGTERM, previous_sigterm) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_benchmark_gate.py b/scripts/test_benchmark_gate.py new file mode 100644 index 00000000..9baa6f27 --- /dev/null +++ b/scripts/test_benchmark_gate.py @@ -0,0 +1,943 @@ +#!/usr/bin/env python3 +"""Focused contract tests for scripts/benchmark_gate.py.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import math +import pathlib +import tempfile +import unittest +from copy import deepcopy +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).with_name("benchmark_gate.py") +SPEC = importlib.util.spec_from_file_location("benchmark_gate", MODULE_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError("cannot load benchmark_gate test module") +gate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate) + + +def fixture(dataset: str = gate.FIXTURE_ID) -> dict: + return { + **gate.fixture_fields(dataset), + "ordered_cases": [ + {"name": name, "seed": 1712 + index * 10} + for index, name in enumerate(gate.EXPECTED_CASES) + ], + } + + +def diagnostic_final_state( + *, + logical_files: int = 1, + logical_bytes: int = 1024, + restored_files: int = 0, +) -> dict: + digest = "d" * 64 + return { + "schema_version": gate.DIAGNOSTIC_SCHEMA_VERSION, + "active_logical_namespace": { + "count": logical_files, + "total_bytes": logical_bytes, + "sha256": digest, + }, + "logical_catalog": {"count": logical_files, "total_bytes": logical_bytes, "sha256": digest}, + "logical_statuses": {"completed": logical_files, "processing": 0, "aborted": 0}, + "chunk_graph": {"count": logical_files, "total_bytes": logical_bytes, "sha256": digest}, + "restored_tree": { + "count": restored_files, + "total_bytes": logical_bytes if restored_files else 0, + "sha256": digest, + }, + "snapshots": {"count": 0, "total_bytes": 0, "sha256": digest}, + "snapshot_count": 0, + "gc": diagnostic_gc(logical_files), + "verification": diagnostic_verification(), + "physical": diagnostic_physical(logical_files, logical_bytes, digest), + "physical_layout_sha256": "e" * 64, + } + + +def diagnostic_gc(logical_files: int) -> dict: + return { + "total_chunks": logical_files, + "reachable_chunks": logical_files, + "unreachable_chunks": 0, + "logically_reclaimable_bytes": 0, + "physically_reclaimable_bytes": 0, + "packed_blocks_live": 1, + "packed_blocks_dead": 0, + "packed_bytes_live": 1024, + "packed_bytes_reclaimable": 0, + "retained_dead_bytes": 0, + } + + +def diagnostic_verification() -> dict: + return { + "blocks_checked": 1, + "physical_hashes_checked": 1, + "compressed_hashes_checked": 0, + "logical_hashes_checked": 1, + "compressed_blocks_checked": 0, + "physical_file_issues": 0, + "snapshot_membership_rows": 0, + "snapshot_reachability_issues": 0, + } + + +def diagnostic_physical(logical_files: int, logical_bytes: int, digest: str) -> dict: + return { + "container_count": 1, + "storage_block_count": 1, + "legacy_block_count": 0, + "chunk_reference_count": logical_files, + "payload_bytes": logical_bytes, + "container_bytes": logical_bytes + 64, + "canonical_sha256": digest, + } + + +def operational_sample(index: int = 0) -> dict[str, int]: + opens = index + 1 + return { + "container_append_count": index + 1, + "container_open_count": opens, + "container_close_count": opens, + "fsync_count": index + 1, + "bytes_written": 1024 * (index + 1), + "bytes_read": 0, + "snapshot_metadata_write_count": 0, + } + + +def raw_row(case: str = "store-large-file", *, workers: int = 4) -> dict: + counters = operational_sample() + return { + "case": case, + "duration_ms": 1000, + "throughput_mbps": 1024 / (1024 * 1024), + "execution": { + "store_folder_workers": workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "execution_stats": { + "total_files": 1, + "total_bytes": 1024, + "workers_used": workers, + "container_append_count": counters["container_append_count"], + "container_open_count": counters["container_open_count"], + "container_close_count": counters["container_close_count"], + "fsync_count": counters["fsync_count"], + "io": { + "container_opens": counters["container_open_count"], + "container_appends": counters["container_append_count"], + "fsyncs": counters["fsync_count"], + "bytes_written": counters["bytes_written"], + "bytes_read": counters["bytes_read"], + }, + }, + "diagnostic_final_state": diagnostic_final_state( + restored_files=1 if case in {"restore-large-file", "restore-many-files"} else 0 + ), + } + + +def raw_report( + *, workers: int = 4, compression: str = "none", dataset: str = gate.FIXTURE_ID +) -> dict: + rows = [raw_row(case, workers=workers) for case in gate.EXPECTED_CASES] + data = { + "schema_version": gate.SCHEMA_VERSION, + "generated_at_utc": "2026-07-25T00:00:00Z", + "dataset": dataset, + "repeat": 1, + "fixture": fixture(dataset), + "execution": { + "store_folder_workers": workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "execution_stats": raw_execution_stats(rows, workers), + "rows": rows, + } + report = {"status": "ok", "command": "benchmark", "data": data} + gate.validate_raw_report( + report, workers=workers, compression=compression, dataset=dataset + ) + return report + + +def raw_execution_stats(rows: list[dict], workers: int) -> dict: + return { + "total_files": sum_execution_stat(rows, "total_files"), + "total_bytes": sum_execution_stat(rows, "total_bytes"), + "workers_used": workers, + "container_append_count": sum_execution_stat(rows, "container_append_count"), + "container_open_count": sum_execution_stat(rows, "container_open_count"), + "container_close_count": sum_execution_stat(rows, "container_close_count"), + "fsync_count": sum_execution_stat(rows, "fsync_count"), + "snapshot_metadata_write_count": 0, + "io": { + field: sum_execution_io_stat(rows, field) + for field in gate.IO_COUNTER_FIELDS + }, + } + + +def sum_execution_stat(rows: list[dict], field: str) -> int: + return sum(row["execution_stats"][field] for row in rows) + + +def sum_execution_io_stat(rows: list[dict], field: str) -> int: + return sum(row["execution_stats"]["io"][field] for row in rows) + + +def aggregate( + durations: list[float] | None = None, + *, + source: str = "a" * 40, + runner_image: str = "image-a", +) -> dict: + durations = durations or [5000, 5000, 5000, 5000, 5000] + cases = [ + aggregate_case(index, name, durations) + for index, name in enumerate(gate.EXPECTED_CASES) + ] + return { + "schema_version": 2, + "evidence_policy_version": gate.EVIDENCE_POLICY_VERSION, + "report_kind": gate.REPORT_KIND, + "status": "ok", + "provenance": aggregate_provenance(source, runner_image), + "profile": { + "codec": "aes-gcm", + "compression": "none", + "dataset": gate.FIXTURE_ID, + "workers": 4, + "pipeline_depth": 1, + "deterministic": True, + }, + "fixture": fixture(), + "warmup_count": 1, + "sample_count": len(durations), + "sample_order": list(range(1, len(durations) + 1)), + "command_durations_ms": [10000] * len(durations), + "command_p95_ms": 10000, + "host_observations": [ + { + "before": {"load_1m": 0, "load_5m": 0, "load_15m": 0, "free_disk_bytes": 1}, + "after": {"load_1m": 0, "load_5m": 0, "load_15m": 0, "free_disk_bytes": 1}, + } + for _ in durations + ], + "operation_totals": gate.operation_totals(len(durations)), + "cleanup_totals": gate.cleanup_totals(len(durations)), + "cases": cases, + } + + +def aggregate_case(index: int, name: str, durations: list[float]) -> dict: + summary = gate.summarize(durations) + logical_bytes = 1024 * (index + 1) + logical_files = index + 1 + restored_files = logical_files if name in {"restore-large-file", "restore-many-files"} else 0 + diagnostic = diagnostic_final_state( + logical_files=logical_files, + logical_bytes=logical_bytes, + restored_files=restored_files, + ) + operational_samples = [operational_sample(index) for _ in durations] + return { + "case": name, + "seed": 1712 + index * 10, + "logical_files": logical_files, + "logical_bytes": logical_bytes, + "workers_used": 4, + "sample_durations_ms": list(durations), + "diagnostic_final_state": diagnostic, + "diagnostic_samples": [deepcopy(diagnostic) for _ in durations], + "operational_samples": operational_samples, + "operational_counter_distributions": gate.summarize_operational_counters( + operational_samples + ), + **summary, + "throughput_mbps": logical_bytes + / (1024 * 1024) + / (summary["median_duration_ms"] / 1000), + } + + +def aggregate_provenance(source: str, runner_image: str) -> dict: + return { + "source_commit": source, + "source_tag": None, + "generated_at_utc": "2026-07-25T00:00:00Z", + "workflow_run_id": "1", + "workflow_job_id": "benchmark", + "workflow_run_attempt": "1", + "runner_os": "Linux", + "runner_image": runner_image, + "runner_arch": "X64", + "cpu_count": 4, + "go_version": "go version go1.25.12 linux/amd64", + "postgres_version": "PostgreSQL 16.14", + "database_image_digest": "sha256:" + "b" * 64, + "binary_sha256": "c" * 64, + } + + +def set_case_durations(report: dict, case_name: str, durations: list[float]) -> None: + case = next(item for item in report["cases"] if item["case"] == case_name) + summary = gate.summarize(durations) + case["sample_durations_ms"] = list(durations) + case.update(summary) + case["throughput_mbps"] = ( + case["logical_bytes"] + / (1024 * 1024) + / (summary["median_duration_ms"] / 1000) + ) + + +class StatisticsTests(unittest.TestCase): + def test_median_mad_cv_and_retained_outlier(self) -> None: + result = gate.summarize([100, 100, 100, 100, 1000]) + self.assertEqual(result["median_duration_ms"], 100) + self.assertEqual(result["mad_ms"], 0) + self.assertGreater(result["coefficient_of_variation_pct"], 0) + + def test_percentile_nearest_rank(self) -> None: + self.assertEqual(gate.percentile_nearest_rank([5, 1, 4, 2, 3], 0.95), 5) + + def test_empty_and_non_finite_samples_fail(self) -> None: + with self.assertRaises(gate.GateError): + gate.summarize([]) + with self.assertRaises(gate.GateError): + gate.summarize([math.inf]) + + +class StrictEvidenceTests(unittest.TestCase): + def test_diagnostic_schema_excludes_sensitive_fields(self) -> None: + state = diagnostic_final_state() + gate.validate_diagnostic_final_state(state, "test") + for forbidden in ("dsn", "password", "username", "database_name", "temporary_path"): + mutated = deepcopy(state) + mutated[forbidden] = "secret" + with self.assertRaisesRegex(gate.GateError, "fields mismatch"): + gate.validate_diagnostic_final_state(mutated, "test") + + def test_hard_final_state_is_separate_from_operational_counters(self) -> None: + as_raw_row = raw_row() + counter_variant = deepcopy(as_raw_row) + counter_variant["execution_stats"]["fsync_count"] = 2 + counter_variant["execution_stats"]["io"]["fsyncs"] = 2 + self.assertEqual(gate.hard_final_state(as_raw_row), gate.hard_final_state(counter_variant)) + self.assertNotEqual( + gate.validate_operational_counters(as_raw_row, workers=4), + gate.validate_operational_counters(counter_variant, workers=4), + ) + + layout_variant = deepcopy(as_raw_row) + layout_variant["diagnostic_final_state"]["physical"]["container_count"] += 1 + layout_variant["diagnostic_final_state"]["physical"]["container_bytes"] += 64 + layout_variant["diagnostic_final_state"]["physical_layout_sha256"] = "a" * 64 + self.assertEqual(gate.hard_final_state(as_raw_row), gate.hard_final_state(layout_variant)) + + state_variant = deepcopy(as_raw_row) + state_variant["diagnostic_final_state"]["active_logical_namespace"]["sha256"] = "f" * 64 + self.assertNotEqual(gate.hard_final_state(as_raw_row), gate.hard_final_state(state_variant)) + self.assertEqual( + gate.validate_operational_counters(as_raw_row, workers=4), + gate.validate_operational_counters(state_variant, workers=4), + ) + + catalog_variant = deepcopy(as_raw_row) + catalog_variant["diagnostic_final_state"]["logical_catalog"]["sha256"] = "a" * 64 + self.assertNotEqual(gate.hard_final_state(as_raw_row), gate.hard_final_state(catalog_variant)) + + def test_logical_status_totals_are_tied_to_catalog_not_namespace(self) -> None: + state = diagnostic_final_state(logical_files=2) + state["active_logical_namespace"]["count"] = 1 + gate.validate_diagnostic_final_state(state, "test") + + state["logical_catalog"]["count"] = 3 + with self.assertRaisesRegex(gate.GateError, "logical catalog count"): + gate.validate_diagnostic_final_state(state, "test") + + def test_trailing_and_repeated_json_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory, "input.json") + path.write_text("{}\n{}\n", encoding="utf-8") + with self.assertRaisesRegex(gate.GateError, "trailing"): + gate.load_json_strict(path) + path.write_text("{", encoding="utf-8") + with self.assertRaisesRegex(gate.GateError, "malformed"): + gate.load_json_strict(path) + + def test_non_finite_json_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory, "input.json") + path.write_text('{"value": NaN}', encoding="utf-8") + with self.assertRaisesRegex(gate.GateError, "non-finite"): + gate.load_json_strict(path) + + def test_legacy_and_empty_reports_fail(self) -> None: + with self.assertRaises(gate.GateError): + gate.validate_aggregate({"data": {"rows": []}}, require_gate_count=True) + report = aggregate() + report["cases"] = [] + with self.assertRaisesRegex(gate.GateError, "non-empty"): + gate.validate_aggregate(report, require_gate_count=True) + for field, value in ( + ("schema_version", 1), + ("report_kind", "wrong"), + ("status", "failed"), + ): + report = aggregate() + report[field] = value + with self.assertRaisesRegex(gate.GateError, "schema/policy/report kind/status"): + gate.validate_aggregate(report, require_gate_count=True) + + def test_duplicate_missing_and_wrong_order_fail(self) -> None: + for mutate in ( + lambda report: report["cases"].pop(), + lambda report: report["cases"].__setitem__(1, report["cases"][0]), + lambda report: report["cases"].reverse(), + lambda report: report["cases"][0].__setitem__("case", "unexpected-case"), + ): + report = aggregate() + mutate(report) + with self.assertRaisesRegex(gate.GateError, "case set/order"): + gate.validate_aggregate(report, require_gate_count=True) + + def test_fixed_sample_order_and_count_are_required(self) -> None: + report = aggregate() + report["sample_order"] = [2, 1, 3, 4, 5] + with self.assertRaisesRegex(gate.GateError, "sample order"): + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + report["sample_count"] = 3 + with self.assertRaisesRegex(gate.GateError, "one warmup and five"): + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + report["warmup_count"] = 2 + with self.assertRaisesRegex(gate.GateError, "one warmup and five"): + gate.validate_aggregate(report, require_gate_count=True) + + def test_derived_throughput_and_statistics_are_recomputed(self) -> None: + report = aggregate() + report["cases"][0]["throughput_mbps"] *= 2 + with self.assertRaisesRegex(gate.GateError, "throughput"): + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + report["cases"][0]["seed"] += 1 + with self.assertRaisesRegex(gate.GateError, "seed mismatch"): + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + report["cases"][0]["logical_files"] = 0 + with self.assertRaisesRegex(gate.GateError, "logical_files"): + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + del report["provenance"]["workflow_run_attempt"] + with self.assertRaisesRegex(gate.GateError, "workflow_run_attempt"): + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + report["fixture"]["many_small_file_count"] += 1 + with self.assertRaisesRegex(gate.GateError, "many_small_file_count"): + gate.validate_aggregate(report, require_gate_count=True) + + def test_functional_sample_failure_is_not_aggregated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + binary = root / "coldkeep" + binary.write_text("#!/usr/bin/env sh\nexit 7\n", encoding="utf-8") + binary.chmod(0o700) + args = argparse.Namespace( + binary=binary, + dataset=gate.FIXTURE_ID, + workers=4, + compression="none", + ) + with self.assertRaisesRegex(gate.GateError, "exit 7"): + gate.capture_sample( + args, + root / "sample.json", + gate.sha256_file(binary), + ) + + report = aggregate() + report["cases"][0]["median_duration_ms"] += 1 + with self.assertRaisesRegex(gate.GateError, "statistic"): + gate.validate_aggregate(report, require_gate_count=True) + + +class OutcomeEPolicyTests(unittest.TestCase): + def test_policy_categories_are_explicit_and_nonempty(self) -> None: + self.assertEqual( + set(gate.FIELD_POLICY), + {"hard_equal", "derived_equal", "bounded_nonnegative", "informational", "excluded_sensitive"}, + ) + self.assertTrue(all(gate.FIELD_POLICY[category] for category in gate.FIELD_POLICY)) + + def test_hard_equal_mutations_fail_closed(self) -> None: + def diagnostic_mutator(section: str, field: str, value: object): + def mutate(report: dict) -> None: + report["cases"][0]["diagnostic_final_state"][section][field] = value + return mutate + + mutations = [ + ("fixture identity", lambda report: report["fixture"].__setitem__("id", "wrong")), + ("seed", lambda report: report["cases"][0].__setitem__("seed", 9999)), + ("active namespace totals", diagnostic_mutator("active_logical_namespace", "count", 2)), + ("active namespace digest", diagnostic_mutator("active_logical_namespace", "sha256", "a" * 64)), + ("logical catalog totals", diagnostic_mutator("logical_catalog", "count", 2)), + ("logical catalog digest", diagnostic_mutator("logical_catalog", "sha256", "a" * 64)), + ("chunk graph", diagnostic_mutator("chunk_graph", "sha256", "a" * 64)), + ("restored tree", diagnostic_mutator("restored_tree", "sha256", "a" * 64)), + ("snapshot membership", diagnostic_mutator("snapshots", "sha256", "a" * 64)), + ("GC state", diagnostic_mutator("gc", "reachable_chunks", 0)), + ("verification", diagnostic_mutator("verification", "blocks_checked", 2)), + ("physical content", diagnostic_mutator("physical", "canonical_sha256", "a" * 64)), + ("physical payload bytes", diagnostic_mutator("physical", "payload_bytes", 2)), + ("operation result", lambda report: report["operation_totals"].__setitem__("failure", 1)), + ("cleanup", lambda report: report["cleanup_totals"].__setitem__("failed", 1)), + ] + for label, mutate in mutations: + with self.subTest(label=label): + report = aggregate() + mutate(report) + with self.assertRaises(gate.GateError): + gate.validate_aggregate(report, require_gate_count=True) + + def test_valid_scheduling_counter_variation_is_retained(self) -> None: + report = aggregate() + case = report["cases"][4] + varied = deepcopy(case["operational_samples"][1]) + varied["container_open_count"] += 1 + varied["container_close_count"] += 1 + varied["fsync_count"] += 1 + case["operational_samples"][1] = varied + case["operational_counter_distributions"] = gate.summarize_operational_counters( + case["operational_samples"] + ) + gate.validate_aggregate(report, require_gate_count=True) + self.assertEqual( + case["operational_counter_distributions"]["container_open_count"]["values"], + [5, 6], + ) + + def test_invalid_operational_counters_fail_closed(self) -> None: + mutations = [ + ("missing", lambda row: row["execution_stats"]["io"].pop("fsyncs")), + ("negative", lambda row: row["execution_stats"]["io"].__setitem__("bytes_read", -1)), + ("wrong type", lambda row: row["execution_stats"]["io"].__setitem__("bytes_read", "0")), + ("non-finite", lambda row: row["execution_stats"]["io"].__setitem__("bytes_read", math.inf)), + ("unbalanced", lambda row: row["execution_stats"].__setitem__("container_close_count", 0)), + ("contradiction", lambda row: ( + row["execution_stats"].__setitem__("container_open_count", 0), + row["execution_stats"].__setitem__("container_close_count", 0), + row["execution_stats"]["io"].__setitem__("container_opens", 0), + )), + ] + for label, mutate in mutations: + with self.subTest(label=label): + row = raw_row() + mutate(row) + with self.assertRaises(gate.GateError): + gate.validate_operational_counters(row, workers=4) + + def test_layout_may_differ_but_canonical_content_may_not(self) -> None: + report = aggregate() + case = report["cases"][0] + case["diagnostic_samples"][1]["physical_layout_sha256"] = "a" * 64 + case["diagnostic_samples"][1]["physical"]["container_count"] += 1 + gate.validate_aggregate(report, require_gate_count=True) + + report = aggregate() + report["cases"][0]["diagnostic_samples"][1]["physical"]["canonical_sha256"] = "a" * 64 + with self.assertRaisesRegex(gate.GateError, "hard diagnostic"): + gate.validate_aggregate(report, require_gate_count=True) + + def test_unknown_and_sensitive_extensions_fail_closed(self) -> None: + mutations = [ + lambda report: report["cases"][0].__setitem__("new_correctness_field", 1), + lambda report: report["cases"][0]["diagnostic_final_state"].__setitem__("new_info", 1), + lambda report: report["provenance"].__setitem__("password", "secret"), + lambda report: report["provenance"].__setitem__("runner_image", "/tmp/private-runner"), + ] + for mutate in mutations: + report = aggregate() + mutate(report) + with self.assertRaises(gate.GateError): + gate.validate_aggregate(report, require_gate_count=True) + + def test_raw_schema_v2_requires_diagnostic_evidence(self) -> None: + report = raw_report() + del report["data"]["rows"][0]["diagnostic_final_state"] + with self.assertRaisesRegex(gate.GateError, "fields mismatch"): + gate.validate_raw_report(report, workers=4, compression="none") + + def test_revalidation_retains_every_sample_without_calibration_claim(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + raw_dir = root / "raw" + gate.write_json(raw_dir / "sample-01.json", raw_report()) + second = raw_report() + second["data"]["rows"][4]["execution_stats"]["container_open_count"] += 1 + second["data"]["rows"][4]["execution_stats"]["container_close_count"] += 1 + second["data"]["rows"][4]["execution_stats"]["fsync_count"] += 1 + second["data"]["rows"][4]["execution_stats"]["io"]["container_opens"] += 1 + second["data"]["rows"][4]["execution_stats"]["io"]["fsyncs"] += 1 + second["data"]["execution_stats"]["container_open_count"] += 1 + second["data"]["execution_stats"]["container_close_count"] += 1 + second["data"]["execution_stats"]["fsync_count"] += 1 + second["data"]["execution_stats"]["io"]["container_opens"] += 1 + second["data"]["execution_stats"]["io"]["fsyncs"] += 1 + gate.write_json(raw_dir / "sample-02.json", second) + output = root / "revalidation.json" + code = gate.revalidate_raw_command(argparse.Namespace( + raw_dir=raw_dir, + compression="none", + workers=4, + output=output, + )) + self.assertEqual(code, 0) + result = gate.load_json_strict(output) + gate.validate_revalidation_report(result) + self.assertEqual(result["sample_count"], 2) + self.assertEqual(result["performance_calibration_status"], "not_evaluated") + self.assertEqual( + result["cases"][4]["operational_counter_distributions"]["fsync_count"]["values"], + [1, 2], + ) + + +class IntegrityCommandTests(unittest.TestCase): + def args(self, root: pathlib.Path, *, dataset: str = "ci-paired-w1-v2") -> argparse.Namespace: + root.mkdir(parents=True, exist_ok=True) + binary = root / "coldkeep" + binary.write_bytes(b"binary") + return argparse.Namespace( + binary=binary, + output_dir=root / "owned" / "integrity", + compression="none", + workers=1, + dataset=dataset, + command_timeout_seconds=600, + source_commit="a" * 40, + source_tag=None, + go_version="go version go1.25.12 linux/amd64", + postgres_version="postgres (PostgreSQL) 16", + database_image_digest="sha256:" + "b" * 64, + ) + + def environment(self) -> dict[str, str]: + return { + "COLDKEEP_CODEC": "aes-gcm", + "DB_HOST": "127.0.0.1", + "DB_PORT": "5432", + "DB_USER": "test", + "DB_PASSWORD": "test", + "DB_NAME": "test", + "DB_SSLMODE": "disable", + } + + def test_integrity_success_owns_output_and_checksums_every_file(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + args = self.args(root) + report = raw_report(workers=1, dataset=args.dataset) + + def capture(_args, path, _binary_hash): + gate.write_json(path, report) + path.with_suffix(".stderr").write_text("", encoding="utf-8") + point = {"load_1m": 0, "load_5m": 0, "load_15m": 0, "free_disk_bytes": 1} + return deepcopy(report), 1000.0, {"before": point, "after": point} + + with mock.patch.dict(gate.os.environ, self.environment(), clear=False), mock.patch.object( + gate, "capture_sample", side_effect=capture + ): + self.assertEqual(gate.integrity_command(args), 0) + result = gate.load_json_strict(args.output_dir / "benchmark-integrity.json") + gate.validate_integrity_report(result) + self.assertEqual(result["classification"], "BENCHMARK_INTEGRITY_PASS") + self.assertEqual(result["completed_sample_count"], 2) + checksum_lines = (args.output_dir / "checksums.sha256").read_text().splitlines() + self.assertEqual(len(checksum_lines), 6) + self.assertTrue(any(line.endswith(" aggregate.json") for line in checksum_lines)) + + def test_integrity_failure_is_checksummed_without_aggregate_claims(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + args = self.args(root) + with mock.patch.dict(gate.os.environ, self.environment(), clear=False), mock.patch.object( + gate, "capture_sample", side_effect=gate.GateError("raw contract mismatch") + ): + self.assertEqual(gate.integrity_command(args), 2) + result = gate.load_json_strict(args.output_dir / "benchmark-integrity.json") + gate.validate_integrity_report(result) + self.assertEqual(result["classification"], "BENCHMARK_INTEGRITY_FAILURE") + self.assertIsNone(result["aggregate_file"]) + self.assertEqual(result["completed_prefix"], []) + self.assertIsNone(result["active_invocation"]) + self.assertEqual(result["incomplete_invocation"]["sample_index"], 1) + self.assertFalse((args.output_dir / "aggregate.json").exists()) + self.assertTrue((args.output_dir / "checksums.sha256").is_file()) + + def test_integrity_rejects_existing_output_and_worker_fixture_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + args = self.args(root) + args.output_dir.mkdir(parents=True) + with self.assertRaisesRegex(gate.GateError, "must not exist"): + gate.integrity_command(args) + args = self.args(root / "second", dataset="ci-paired-w4-v2") + with self.assertRaisesRegex(gate.GateError, "worker profile mismatch"): + gate.integrity_command(args) + + +class ComparisonTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temp.name) + self.thresholds = self.root / "thresholds.yaml" + self.thresholds.write_text( + """ +defaults: + uncompressed: + duration_regression_pct: 5 + compressed: + duration_regression_warning_pct: 15 +per_case_overrides: + uncompressed: + snapshot-creation: + duration_regression_pct: 3 + compressed: + store-many-small-files: + duration_regression_warning_pct: 20 +""".lstrip(), + encoding="utf-8", + ) + + def tearDown(self) -> None: + self.temp.cleanup() + + def run_compare(self, baseline: dict, candidate: dict) -> tuple[int, dict]: + baseline_path = self.root / "baseline.json" + candidate_path = self.root / "candidate.json" + output_path = self.root / "comparison.json" + gate.write_json(baseline_path, baseline) + gate.write_json(candidate_path, candidate) + code = gate.compare_command( + argparse.Namespace( + baseline=baseline_path, + candidate=candidate_path, + thresholds=self.thresholds, + mode="uncompressed", + manifest=None, + output=output_path, + ) + ) + return code, gate.load_json_strict(output_path) + + def test_exact_threshold_passes_and_above_threshold_fails(self) -> None: + candidate = aggregate() + set_case_durations(candidate, "store-large-file", [5250] * 5) + code, report = self.run_compare(aggregate(), candidate) + self.assertEqual(code, 0) + store = next(item for item in report["outcomes"] if item["case"] == "store-large-file") + self.assertEqual(store["classification"], "pass") + + candidate = aggregate() + set_case_durations(candidate, "store-large-file", [5251] * 5) + code, report = self.run_compare(aggregate(), candidate) + self.assertEqual(code, 1) + store = next(item for item in report["outcomes"] if item["case"] == "store-large-file") + self.assertEqual(store["classification"], "PERFORMANCE_REGRESSION") + + def test_high_variability_precedes_regression(self) -> None: + candidate = aggregate([5000, 5000, 5300, 5600, 5900]) + code, report = self.run_compare(aggregate(), candidate) + self.assertEqual(code, 1) + self.assertTrue( + all(item["classification"] == "BENCHMARK_UNSTABLE" for item in report["outcomes"]) + ) + + def test_one_extreme_outlier_is_retained_without_moving_median_or_mad(self) -> None: + code, report = self.run_compare(aggregate(), aggregate([5000, 5000, 5000, 5000, 9000])) + self.assertEqual(code, 0) + self.assertTrue(all(item["classification"] == "pass" for item in report["outcomes"])) + + def test_environment_and_fixture_mismatch_fail_closed(self) -> None: + baseline = aggregate() + candidate = aggregate() + candidate["provenance"]["go_version"] = "go version go1.25.13 linux/amd64" + with self.assertRaisesRegex(gate.GateError, "go_version"): + self.run_compare(baseline, candidate) + + candidate = aggregate() + candidate["cases"][0]["logical_files"] += 1 + with self.assertRaises(gate.GateError): + self.run_compare(baseline, candidate) + + def test_runner_image_drift_is_warning_only(self) -> None: + code, report = self.run_compare( + aggregate(runner_image="image-a"), + aggregate(runner_image="image-b"), + ) + self.assertEqual(code, 0) + self.assertEqual(report["warnings"], ["resolved runner image differs from baseline"]) + + +class ManifestTests(unittest.TestCase): + def test_manifest_hash_validation_and_stale_hash_failure(self) -> None: + with tempfile.TemporaryDirectory(dir=pathlib.Path.cwd()) as directory: + root = pathlib.Path(directory) + thresholds = root / "thresholds.yaml" + thresholds.write_text("thresholds\n", encoding="utf-8") + baselines = [] + for profile, (compression, workers) in gate.MANIFEST_PROFILES.items(): + path = root / f"baseline-{profile}.json" + report = aggregate() + report["profile"]["compression"] = compression + report["profile"]["workers"] = workers + for case in report["cases"]: + case["workers_used"] = workers + gate.write_json(path, report) + baselines.append( + f"{profile}={path.relative_to(pathlib.Path.cwd())}" + ) + manifest = root / "manifest.json" + code = gate.manifest_command( + argparse.Namespace( + baseline=baselines, + thresholds=thresholds.relative_to(pathlib.Path.cwd()), + output=manifest, + ) + ) + self.assertEqual(code, 0) + self.assertEqual( + gate.validate_manifest_command(argparse.Namespace(manifest=manifest)), + 0, + ) + pathlib.Path(baselines[0].split("=", 1)[1]).write_text("{}\n", encoding="utf-8") + with self.assertRaisesRegex(gate.GateError, "hash mismatch"): + gate.validate_manifest_command(argparse.Namespace(manifest=manifest)) + + +class CalibrationTests(unittest.TestCase): + def test_fixed_matrix_passes_and_short_case_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + thresholds = root / "thresholds.yaml" + thresholds.write_text( + """ +defaults: + uncompressed: + duration_regression_pct: 5 + compressed: + duration_regression_warning_pct: 15 +per_case_overrides: + uncompressed: + snapshot-creation: + duration_regression_pct: 3 + compressed: + store-many-small-files: + duration_regression_warning_pct: 20 +""".lstrip(), + encoding="utf-8", + ) + inputs = [] + for compression in ("none", "zstd"): + for workers in (1, 4): + for replicate in (1, 2): + report = aggregate([5000] * 10) + report["profile"]["compression"] = compression + report["profile"]["workers"] = workers + for case in report["cases"]: + case["workers_used"] = workers + path = root / f"{compression}-w{workers}-r{replicate}.json" + gate.write_json(path, report) + inputs.append(f"{compression}-w{workers}-r{replicate}={path}") + output = root / "calibration.json" + code = gate.calibration_command( + argparse.Namespace( + aggregate=inputs, + thresholds=thresholds, + output=output, + ) + ) + self.assertEqual(code, 0) + + first_path = pathlib.Path(inputs[0].split("=", 1)[1]) + report = gate.load_json_strict(first_path) + set_case_durations(report, "store-large-file", [4999] * 10) + gate.write_json(first_path, report) + code = gate.calibration_command( + argparse.Namespace( + aggregate=inputs, + thresholds=thresholds, + output=output, + ) + ) + self.assertEqual(code, 1) + self.assertTrue(gate.load_json_strict(output)["failures"]) + + # The three inclusive acceptance boundaries pass exactly. + report = gate.load_json_strict(first_path) + set_case_durations(report, "store-large-file", [5000] * 10) + report["command_durations_ms"] = [120000] * 10 + report["command_p95_ms"] = 120000 + gate.write_json(first_path, report) + second_path = pathlib.Path(inputs[1].split("=", 1)[1]) + second = gate.load_json_strict(second_path) + set_case_durations(second, "store-large-file", [5250] * 10) + gate.write_json(second_path, second) + code = gate.calibration_command( + argparse.Namespace(aggregate=inputs, thresholds=thresholds, output=output) + ) + self.assertEqual(code, 0) + + # Strictly exceeding either 120 seconds or 5% fails. + report["command_durations_ms"][-1] = 120001 + report["command_p95_ms"] = 120001 + gate.write_json(first_path, report) + code = gate.calibration_command( + argparse.Namespace(aggregate=inputs, thresholds=thresholds, output=output) + ) + self.assertEqual(code, 1) + report["command_durations_ms"][-1] = 120000 + report["command_p95_ms"] = 120000 + gate.write_json(first_path, report) + set_case_durations(second, "store-large-file", [5251] * 10) + gate.write_json(second_path, second) + code = gate.calibration_command( + argparse.Namespace(aggregate=inputs, thresholds=thresholds, output=output) + ) + self.assertEqual(code, 1) + + # Fixed odd/even partitions are evaluated independently. + set_case_durations(second, "store-large-file", [5000, 5000, 5200, 5000, 5400, 5000, 5600, 5000, 5800, 5000]) + gate.write_json(second_path, second) + code = gate.calibration_command( + argparse.Namespace(aggregate=inputs, thresholds=thresholds, output=output) + ) + self.assertEqual(code, 1) + self.assertTrue( + any("odd five-sample MAD ratio" in item for item in gate.load_json_strict(output)["failures"]) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_paired_benchmark_gate.py b/scripts/test_paired_benchmark_gate.py new file mode 100644 index 00000000..c56066c6 --- /dev/null +++ b/scripts/test_paired_benchmark_gate.py @@ -0,0 +1,2105 @@ +#!/usr/bin/env python3 +"""Contract tests for scripts/paired_benchmark_gate.py.""" + +from __future__ import annotations + +import argparse +import importlib.util +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest +from copy import deepcopy +from unittest import mock + +SCRIPT_DIR = pathlib.Path(__file__).parent +sys.path.insert(0, str(SCRIPT_DIR)) +MODULE_PATH = SCRIPT_DIR / "paired_benchmark_gate.py" +SPEC = importlib.util.spec_from_file_location("paired_benchmark_gate", MODULE_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError("cannot load paired_benchmark_gate test module") +gate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate) + + +class FakeProcess: + def __init__(self, *, returncode: int = 0, stdout: str = "", stderr: str = "", timeout: bool = False): + self.returncode = None if timeout else returncode + self._final_returncode = returncode + self._stdout = stdout + self._stderr = stderr + self._timeout = timeout + self._communicates = 0 + self.timeout_values: list[float | None] = [] + self.pid = 424242 + + def poll(self) -> int | None: + return self.returncode + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + self._communicates += 1 + self.timeout_values.append(timeout) + if self._timeout and self._communicates == 1: + raise subprocess.TimeoutExpired(["coldkeep"], timeout or 0) + self.returncode = self._final_returncode + return self._stdout, self._stderr + + +def fixture(dataset: str = "ci-paired-w1-v1") -> dict: + expected = {key: value for key, value in gate.FIXTURES[dataset].items() if key != "workers"} + return { + **expected, + "ordered_cases": [ + {"name": name, "seed": 1712 + index * 10} + for index, name in enumerate(gate.ORDERED_CASES) + ], + } + + +def diagnostic_state(*, restored: bool = False) -> dict: + digest = "d" * 64 + return { + "schema_version": 2, + "active_logical_namespace": {"count": 1, "total_bytes": 1024, "sha256": digest}, + "logical_catalog": {"count": 1, "total_bytes": 1024, "sha256": digest}, + "logical_statuses": {"completed": 1, "processing": 0, "aborted": 0}, + "chunk_graph": {"count": 1, "total_bytes": 1024, "sha256": digest}, + "restored_tree": { + "count": 1 if restored else 0, + "total_bytes": 1024 if restored else 0, + "sha256": digest, + }, + "snapshots": {"count": 0, "total_bytes": 0, "sha256": digest}, + "snapshot_count": 0, + "gc": { + "total_chunks": 1, + "reachable_chunks": 1, + "unreachable_chunks": 0, + "logically_reclaimable_bytes": 0, + "physically_reclaimable_bytes": 0, + "packed_blocks_live": 1, + "packed_blocks_dead": 0, + "packed_bytes_live": 1024, + "packed_bytes_reclaimable": 0, + "retained_dead_bytes": 0, + }, + "verification": { + "blocks_checked": 1, + "physical_hashes_checked": 1, + "compressed_hashes_checked": 0, + "logical_hashes_checked": 1, + "compressed_blocks_checked": 0, + "physical_file_issues": 0, + "snapshot_membership_rows": 0, + "snapshot_reachability_issues": 0, + }, + "physical": { + "container_count": 1, + "storage_block_count": 1, + "legacy_block_count": 0, + "chunk_reference_count": 1, + "payload_bytes": 1024, + "container_bytes": 1088, + "canonical_sha256": digest, + }, + "physical_layout_sha256": "e" * 64, + } + + +def raw_row(case_name: str, *, workers: int, duration_ms: float) -> dict: + return { + "case": case_name, + "duration_ms": duration_ms, + "throughput_mbps": 1024 / (1024 * 1024) / (duration_ms / 1000), + "execution": { + "store_folder_workers": workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "execution_stats": { + "total_files": 1, + "total_bytes": 1024, + "workers_used": workers, + "container_append_count": 1, + "fsync_count": 1, + "container_open_count": 1, + "container_close_count": 1, + "io": { + "container_opens": 1, + "container_appends": 1, + "fsyncs": 1, + "bytes_written": 1024, + "bytes_read": 0, + }, + }, + "diagnostic_final_state": diagnostic_state( + restored=case_name in {"restore-large-file", "restore-many-files"} + ), + } + + +def recompute_top(report: dict) -> None: + rows = report["data"]["rows"] + report["data"]["execution_stats"] = { + "total_files": sum_row_execution_stat(rows, "total_files"), + "total_bytes": sum_row_execution_stat(rows, "total_bytes"), + "workers_used": report["data"]["execution"]["store_folder_workers"], + "container_append_count": sum_row_execution_stat(rows, "container_append_count"), + "fsync_count": sum_row_execution_stat(rows, "fsync_count"), + "container_open_count": sum_row_execution_stat(rows, "container_open_count"), + "container_close_count": sum_row_execution_stat(rows, "container_close_count"), + "snapshot_metadata_write_count": 0, + "io": { + field: sum_row_io_stat(rows, field) + for field in gate.raw_gate.IO_COUNTER_FIELDS + }, + } + + +def sum_row_execution_stat(rows: list[dict], field: str) -> int: + return sum(row["execution_stats"][field] for row in rows) + + +def sum_row_io_stat(rows: list[dict], field: str) -> int: + return sum(row["execution_stats"]["io"][field] for row in rows) + + +def raw_report( + *, dataset: str = "ci-paired-w1-v1", workers: int = 1, duration_ms: float = 1000 +) -> dict: + rows = [raw_row(case_name, workers=workers, duration_ms=duration_ms) for case_name in gate.ORDERED_CASES] + report = { + "status": "ok", + "command": "benchmark", + "data": { + "schema_version": 2, + "generated_at_utc": "2026-07-27T00:00:00Z", + "dataset": dataset, + "repeat": 1, + "fixture": fixture(dataset), + "execution": { + "store_folder_workers": workers, + "pipeline_depth": 1, + "deterministic": True, + }, + "execution_stats": {}, + "rows": rows, + }, + } + recompute_top(report) + gate.validate_raw_report(report, dataset=dataset, workers=workers, compression="none") + return report + + +def records_for_ratios( + ratios: list[float], *, dataset: str = "ci-paired-w1-v1", workers: int = 1 +) -> list[dict]: + records = [] + for ordinal, pair_order in enumerate(gate.measured_order(len(ratios)), start=1): + for position, side in enumerate(pair_order, start=1): + duration = 1000 if side == "reference" else 1000 * ratios[ordinal - 1] + records.append( + { + "pair_ordinal": ordinal, + "position": position, + "side": side, + "envelope": raw_report( + dataset=dataset, workers=workers, duration_ms=duration + ), + } + ) + return records + + +def thresholds(value: float = 5.0) -> dict[str, float]: + return {case_name: value for case_name in gate.PERFORMANCE_CASES} + + +def manifest(reference_sha: str = "a" * 40) -> dict: + return { + "schema_version": 1, + "report_kind": gate.REFERENCE_MANIFEST_KIND, + "release_train": "v1.13", + "reference_sha": reference_sha, + "approval": {"kind": "trusted_tag", "value": "v1.13.11"}, + "contract_version": gate.CONTRACT_VERSION, + "raw_schema_version": 2, + "diagnostic_schema_version": 2, + "fixtures": sorted(gate.FIXTURES), + "ordered_cases": list(gate.ORDERED_CASES), + "performance_cases": list(gate.PERFORMANCE_CASES), + "execution_order": { + "warmups": list(gate.WARMUP_ORDER), + "measured_pairs": [list(pair) for pair in gate.FIVE_PAIR_ORDER], + }, + "pair_count": 5, + "threshold_policy_id": "paired-v1-test", + "threshold_policy_sha256": "b" * 64, + } + + +def invocation_inventory(pair_count: int) -> list[dict]: + items = [] + for position, side in enumerate(gate.WARMUP_ORDER, start=1): + items.append(invocation("warmup", None, position, side)) + for ordinal, pair_order in enumerate(gate.measured_order(pair_count), start=1): + for position, side in enumerate(pair_order, start=1): + items.append(invocation("measured", ordinal, position, side)) + return items + + +def invocation(kind: str, ordinal: int | None, position: int, side: str) -> dict: + if kind == "warmup": + raw_file = f"raw/warmup-{position:02d}-{side}.json" + else: + raw_file = f"raw/pair-{ordinal:02d}/{position:02d}-{side}.json" + return { + "kind": kind, + "pair_ordinal": ordinal, + "position": position, + "side": side, + "raw_file": raw_file, + "stderr_file": pathlib.PurePosixPath(raw_file).with_suffix(".stderr").as_posix(), + "command_duration_ms": 1000, + "binary_sha256": ("c" if side == "reference" else "d") * 64, + "host_observation": { + "before": {"load_1m": 0, "load_5m": 0, "load_15m": 0, "cpu_count": 4}, + "after": {"load_1m": 0, "load_5m": 0, "load_15m": 0, "cpu_count": 4}, + }, + } + + +def report_summary( + *, profile: str = "none-w1", classification: str | None = None, mode: str = "production" +) -> dict: + compression, workers, dataset = gate.PROFILE_MATRIX[profile] + pair_count = 5 if mode == "production" else 10 + if classification is None: + classification = "PASS" if mode == "production" else "DIAGNOSTIC_QUALIFIED" + cases = summary_cases(pair_count, mode) + distributions = summary_distributions() + inventory = invocation_inventory(pair_count) + if mode == "diagnostic": + for item in inventory: + item["binary_sha256"] = "c" * 64 + return { + "schema_version": 1, + "evidence_policy_version": 2, + "report_kind": gate.REPORT_KIND, + "status": "complete", + "mode": mode, + "classification": classification, + "contract_version": gate.CONTRACT_VERSION, + "authority": gate.authority_contract(mode), + "identity": summary_identity(mode), + "governance": summary_governance(mode), + "profile": summary_profile(compression, dataset, workers), + "fixture": fixture(dataset), + "warmup_order": list(gate.WARMUP_ORDER), + "measured_order": [list(pair) for pair in gate.measured_order(pair_count)], + "pair_count": pair_count, + "profile_elapsed_ms": 1000, + "invocation_inventory": inventory, + "cases": cases, + "operational_counter_distributions": distributions, + "hard_state_comparison": {"status": "equal", "case_count": 9}, + "cleanup": summary_cleanup(pair_count), + "provenance": summary_provenance(), + } + + +def summary_cases(pair_count: int, mode: str) -> list[dict]: + cases = [] + for case_name in gate.ORDERED_CASES: + if case_name not in gate.PERFORMANCE_CASES: + cases.append({"case": case_name, "performance_gated": False}) + else: + cases.append(performance_summary_case(case_name, pair_count, mode)) + return cases + + +def performance_summary_case(case_name: str, pair_count: int, mode: str) -> dict: + return { + "case": case_name, + "performance_gated": True, + "paired_ratios": [1.0] * pair_count, + "median_ratio": 1.0, + "regression_pct": 0.0, + "paired_mad_ratio_pct": 0.0, + "stability_boundary_pct": 2.5, + "threshold_pct": 5.0 if mode == "production" else None, + "candidate_throughput_mbps": 1.0, + "status": "pass", + } + + +def summary_distributions() -> dict: + return { + side: { + case_name: { + field: {"min": 0, "max": 0, "values": [0]} + for field in gate.raw_gate.OPERATIONAL_COUNTER_FIELDS + } + for case_name in gate.ORDERED_CASES + } + for side in ("reference", "candidate") + } + + +def summary_identity(mode: str) -> dict: + return { + "reference_sha": "a" * 40, + "candidate_sha": "b" * 40, + "reference_binary_sha256": "c" * 64, + "candidate_binary_sha256": ("d" if mode == "production" else "c") * 64, + } + + +def summary_governance(mode: str) -> dict: + if mode == "production": + return { + "status": "governed", + "manifest_sha256": "e" * 64, + "threshold_policy_id": "paired-v1-test", + "threshold_policy_sha256": "f" * 64, + } + return { + "status": "provisional-diagnostic", + "manifest_sha256": None, + "threshold_policy_id": None, + "threshold_policy_sha256": None, + } + + +def summary_profile(compression: str, dataset: str, workers: int) -> dict: + return { + "codec": "aes-gcm", + "compression": compression, + "dataset": dataset, + "workers": workers, + "pipeline_depth": 1, + "deterministic": True, + } + + +def summary_cleanup(pair_count: int) -> dict: + attempts = (2 + pair_count * 2) * len(gate.ORDERED_CASES) + return {"status": "complete", "attempted": attempts, "succeeded": attempts, "failed": 0} + + +def summary_provenance() -> dict: + return { + "event_name": "pull_request", + "repository_id": "owner/coldkeep", + "runner_os": "Linux", + "runner_image": "image", + "runner_arch": "X64", + "cpu_count": 4, + "go_version": "go1.25.1", + "postgres_version": "16.14", + "database_image_digest": "sha256:" + "1" * 64, + } + + +def write_summary_artifact(directory: pathlib.Path, report: dict) -> None: + directory.mkdir() + for invocation_record in report["invocation_inventory"]: + raw_path = directory / invocation_record["raw_file"] + stderr_path = directory / invocation_record["stderr_file"] + raw_path.parent.mkdir(parents=True, exist_ok=True) + raw_path.write_text("{}\n", encoding="utf-8") + stderr_path.write_text("", encoding="utf-8") + gate.raw_gate.write_json(directory / "paired-comparison.json", report) + gate._write_checksums(directory) + + +def write_governed_repository(repository: pathlib.Path) -> None: + governance = repository / "benchmarks" / "paired" + governance.mkdir(parents=True) + policy = { + "schema_version": 1, + "report_kind": gate.THRESHOLD_POLICY_KIND, + "contract_version": gate.CONTRACT_VERSION, + "policy_id": "paired-v1-test", + "cases": thresholds(5), + } + policy_path = governance / "threshold-policy-v1.13.json" + gate.raw_gate.write_json(policy_path, policy) + governed_manifest = manifest("a" * 40) + governed_manifest["approval"] = {"kind": "reviewed_record", "value": "test-approval"} + governed_manifest["threshold_policy_sha256"] = gate._binary_hash(policy_path) + gate.raw_gate.write_json(governance / "reference-v1.13.json", governed_manifest) + + +def write_complete_profile_artifact( + directory: pathlib.Path, repository: pathlib.Path, *, profile: str = "none-w1" +) -> None: + compression, workers, dataset = gate.PROFILE_MATRIX[profile] + measured = records_for_ratios([1.0] * 5, dataset=dataset, workers=workers) + warmups = profile_warmup_records(dataset, workers) + comparison = gate.compare_records( + measured, + pair_count=5, + dataset=dataset, + workers=workers, + compression=compression, + mode="production", + thresholds=thresholds(5), + ) + report = report_summary(profile=profile) + apply_comparison_summary(report, comparison) + + directory.mkdir() + report["governance"] = copy_profile_governance(directory, repository) + write_profile_observations(directory, report, warmups + measured) + gate.raw_gate.write_json(directory / "paired-comparison.json", report) + gate._write_checksums(directory) + + +def write_complete_diagnostic_profile_artifact( + directory: pathlib.Path, + *, + profile: str = "none-w1", + ratios: list[float] | None = None, + elapsed_ms: float = 1000, +) -> None: + compression, workers, dataset = gate.PROFILE_MATRIX[profile] + measured = records_for_ratios( + ratios or [1.0] * 10, dataset=dataset, workers=workers + ) + warmups = profile_warmup_records(dataset, workers) + comparison = gate.compare_records( + measured, + pair_count=10, + dataset=dataset, + workers=workers, + compression=compression, + mode="diagnostic", + ) + report = report_summary(profile=profile, mode="diagnostic") + report["profile_elapsed_ms"] = elapsed_ms + report["classification"] = comparison["classification"] + if ( + elapsed_ms > gate.DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + and report["classification"] == "DIAGNOSTIC_QUALIFIED" + ): + report["classification"] = "DIAGNOSTIC_REJECTED" + apply_comparison_summary(report, comparison) + + directory.mkdir() + write_profile_observations(directory, report, warmups + measured) + gate.raw_gate.write_json(directory / "paired-comparison.json", report) + gate._write_checksums(directory) + + +def profile_warmup_records(dataset: str, workers: int) -> list[dict]: + return [ + { + "kind": "warmup", + "pair_ordinal": None, + "position": position, + "side": side, + "envelope": raw_report(dataset=dataset, workers=workers), + } + for position, side in enumerate(gate.WARMUP_ORDER, start=1) + ] + + +def apply_comparison_summary(report: dict, comparison: dict) -> None: + report["fixture"] = comparison["fixture"] + report["cases"] = comparison["cases"] + report["operational_counter_distributions"] = comparison[ + "operational_counter_distributions" + ] + report["hard_state_comparison"] = comparison["hard_state_comparison"] + + +def copy_profile_governance(directory: pathlib.Path, repository: pathlib.Path) -> dict: + artifact_governance = directory / "governance" + artifact_governance.mkdir() + repository_governance = repository / "benchmarks" / "paired" + shutil.copyfile( + repository_governance / "reference-v1.13.json", + artifact_governance / "reference-manifest.json", + ) + shutil.copyfile( + repository_governance / "threshold-policy-v1.13.json", + artifact_governance / "threshold-policy.json", + ) + return { + "status": "governed", + "manifest_sha256": gate._binary_hash(artifact_governance / "reference-manifest.json"), + "threshold_policy_id": "paired-v1-test", + "threshold_policy_sha256": gate._binary_hash( + artifact_governance / "threshold-policy.json" + ), + } + + +def write_profile_observations( + directory: pathlib.Path, report: dict, observations: list[dict] +) -> None: + for invocation_record, observation in zip(report["invocation_inventory"], observations): + raw_path = directory / invocation_record["raw_file"] + stderr_path = directory / invocation_record["stderr_file"] + raw_path.parent.mkdir(parents=True, exist_ok=True) + gate.raw_gate.write_json(raw_path, observation["envelope"]) + stderr_path.write_text("", encoding="utf-8") + + +class PairedContractTests(unittest.TestCase): + def test_fixture_registry_and_performance_case_list_are_exact(self) -> None: + self.assertEqual( + gate.FIXTURES, + { + "ci-paired-w1-v1": { + "id": "ci-paired-w1-v1", + "seed": 1701, + "large_file_size_bytes": 96 * 1024 * 1024, + "many_small_file_count": 600, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 400, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 1, + }, + "ci-paired-w4-v1": { + "id": "ci-paired-w4-v1", + "seed": 1701, + "large_file_size_bytes": 128 * 1024 * 1024, + "many_small_file_count": 1200, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 800, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 4, + }, + "ci-paired-w1-v2": { + "id": "ci-paired-w1-v2", + "seed": 1701, + "large_file_size_bytes": 64 * 1024 * 1024, + "many_small_file_count": 400, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 400, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 1, + }, + "ci-paired-w4-v2": { + "id": "ci-paired-w4-v2", + "seed": 1701, + "large_file_size_bytes": 64 * 1024 * 1024, + "many_small_file_count": 400, + "many_small_file_size_bytes": 1024, + "mixed_file_count": 800, + "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, + "remove_every": 4, + "case_database_isolation": True, + "workers": 4, + }, + }, + ) + self.assertEqual( + gate.PERFORMANCE_CASES, + ( + "store-large-file", + "store-many-small-files", + "restore-many-files", + "snapshot-creation", + "gc-after-churn", + "stats-inspect", + "verify-system-deep", + ), + ) + self.assertEqual( + set(gate.ORDERED_CASES) - set(gate.PERFORMANCE_CASES), + {"store-mixed-dataset", "restore-large-file"}, + ) + + def test_governed_artifact_names(self) -> None: + self.assertEqual( + gate.profile_artifact_name( + candidate_sha="b" * 40, + reference_sha="a" * 40, + compression="zstd", + workers=4, + attempt=2, + ), + "benchmark-paired-bbbbbbbbbbbb-against-aaaaaaaaaaaa-zstd-w4-a2", + ) + self.assertEqual( + gate.decision_artifact_name( + candidate_sha="b" * 40, reference_sha="a" * 40, attempt=2 + ), + "benchmark-paired-bbbbbbbbbbbb-against-aaaaaaaaaaaa-decision-a2", + ) + + def test_fixed_warmup_and_pair_orders(self) -> None: + self.assertEqual(gate.WARMUP_ORDER, ("candidate", "reference")) + self.assertEqual( + gate.measured_order(5), + ( + ("reference", "candidate"), + ("candidate", "reference"), + ("candidate", "reference"), + ("reference", "candidate"), + ("reference", "candidate"), + ), + ) + self.assertEqual( + gate.measured_order(10), + ( + ("reference", "candidate"), + ("candidate", "reference"), + ("candidate", "reference"), + ("reference", "candidate"), + ("reference", "candidate"), + ("candidate", "reference"), + ("reference", "candidate"), + ("reference", "candidate"), + ("candidate", "reference"), + ("candidate", "reference"), + ), + ) + self.assertEqual( + [pair[0] for pair in gate.measured_order(10)].count("reference"), 5 + ) + self.assertEqual( + [pair[0] for pair in gate.measured_order(10)].count("candidate"), 5 + ) + with self.assertRaises(gate.PairedGateError) as caught: + gate.measured_order(6) + self.assertEqual(caught.exception.classification, "PAIR_INVENTORY_INVALID") + + def test_warmups_are_ordered_and_participate_in_hard_state_validation(self) -> None: + measured = records_for_ratios([1.0] * 5) + warmups = [ + { + "kind": "warmup", + "pair_ordinal": None, + "position": position, + "side": side, + "envelope": raw_report(), + } + for position, side in enumerate(gate.WARMUP_ORDER, start=1) + ] + gate.validate_warmups( + warmups, + measured, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + ) + warmups[0]["envelope"]["data"]["rows"][0]["diagnostic_final_state"][ + "active_logical_namespace" + ]["sha256"] = "a" * 64 + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_warmups( + warmups, + measured, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + ) + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + + def test_paired_ratio_uses_pairwise_values(self) -> None: + result = gate.compare_records( + records_for_ratios([1.00, 1.01, 1.02, 1.03, 2.00]), + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(5), + ) + case = result["cases"][0] + self.assertEqual(case["paired_ratios"], [1.0, 1.01, 1.02, 1.03, 2.0]) + self.assertEqual(case["median_ratio"], 1.02) + self.assertAlmostEqual(case["regression_pct"], 2.0) + self.assertAlmostEqual(case["paired_mad_ratio_pct"], 0.01 / 1.02 * 100) + + def test_exact_threshold_passes_and_above_fails(self) -> None: + exact = gate.compare_records( + records_for_ratios([1.05] * 5), + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(5), + ) + self.assertEqual(exact["classification"], "PASS") + above = gate.compare_records( + records_for_ratios([1.05001] * 5), + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(5), + ) + self.assertEqual(above["classification"], "PERFORMANCE_REGRESSION") + + def test_instability_precedes_regression(self) -> None: + result = gate.compare_records( + records_for_ratios([1.0, 1.05, 1.1, 1.15, 1.2]), + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(5), + ) + self.assertEqual(result["classification"], "BENCHMARK_ENVIRONMENT_UNSTABLE") + + def test_diagnostic_mad_exact_boundary_passes_and_above_fails(self) -> None: + exact_ratios = [0.95, 0.95, 0.975, 0.975, 1.0, 1.0, 1.025, 1.025, 1.05, 1.05] + exact = gate.compare_records( + records_for_ratios(exact_ratios), + pair_count=10, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + self.assertEqual(exact["classification"], "DIAGNOSTIC_QUALIFIED") + self.assertEqual(exact["cases"][0]["paired_mad_ratio_pct"], 2.5) + + above_ratios = [0.948, 0.948, 0.974, 0.974, 1.0, 1.0, 1.026, 1.026, 1.052, 1.052] + above = gate.compare_records( + records_for_ratios(above_ratios), + pair_count=10, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + self.assertEqual(above["classification"], "BENCHMARK_ENVIRONMENT_UNSTABLE") + self.assertGreater(above["cases"][0]["paired_mad_ratio_pct"], 2.5) + + def test_diagnostic_requires_ten_pairs_and_five_percent_signal(self) -> None: + with self.assertRaises(gate.PairedGateError): + gate.compare_records( + records_for_ratios([1.0] * 5), + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + result = gate.compare_records( + records_for_ratios([1.051] * 10), + pair_count=10, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + self.assertEqual(result["classification"], "DIAGNOSTIC_REJECTED") + for ratio in (0.95, 1.05): + with self.subTest(exact_ratio=ratio): + result = gate.compare_records( + records_for_ratios([ratio] * 10), + pair_count=10, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + self.assertEqual(result["classification"], "DIAGNOSTIC_QUALIFIED") + for ratio in (0.949999, 1.050001): + with self.subTest(outside_ratio=ratio): + result = gate.compare_records( + records_for_ratios([ratio] * 10), + pair_count=10, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + self.assertEqual(result["classification"], "DIAGNOSTIC_REJECTED") + + def test_diagnostic_pair_inventory_rejects_missing_duplicate_and_wrong_order(self) -> None: + base = records_for_ratios([1.0] * 10) + mutations = (base[:-2], base + base[-2:], base[:2] + list(reversed(base[2:4])) + base[4:]) + for mutated in mutations: + with self.subTest(invocations=len(mutated)): + with self.assertRaises(gate.PairedGateError) as caught: + gate.compare_records( + mutated, + pair_count=10, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="diagnostic", + ) + self.assertEqual(caught.exception.classification, "PAIR_INVENTORY_INVALID") + for pair_count in (9, 11): + report = report_summary(mode="diagnostic") + report["pair_count"] = pair_count + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "PAIR_INVENTORY_INVALID") + + def test_diagnostic_profile_duration_boundary_and_precedence(self) -> None: + report = report_summary(mode="diagnostic") + report["profile_elapsed_ms"] = gate.DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + gate.validate_report_summary(report, expected_profile="none-w1") + + report = report_summary(mode="diagnostic") + report["profile_elapsed_ms"] = gate.DIAGNOSTIC_MAX_PROFILE_ELAPSED_MS + 0.001 + report["classification"] = "DIAGNOSTIC_REJECTED" + gate.validate_report_summary(report, expected_profile="none-w1") + + report["hard_state_comparison"]["status"] = "mismatch" + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "CORRECTNESS_REGRESSION") + + report = report_summary(mode="diagnostic") + report["cases"][0]["paired_ratios"] = [1.051] * 10 + report["cases"][0]["median_ratio"] = 1.051 + report["cases"][0]["regression_pct"] = 5.1 + report["cases"][0]["status"] = "qualification_rejected" + report["classification"] = "DIAGNOSTIC_REJECTED" + report["cleanup"]["failed"] = 1 + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + + def test_missing_duplicate_and_reordered_pairs_fail_closed(self) -> None: + base = records_for_ratios([1.0] * 5) + for mutated in (base[:-1], base[:1] + base[:1] + base[2:], list(reversed(base))): + with self.assertRaises(gate.PairedGateError) as caught: + gate.compare_records( + mutated, + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(), + ) + self.assertEqual(caught.exception.classification, "PAIR_INVENTORY_INVALID") + + def test_layout_and_counter_variation_are_permitted(self) -> None: + records = records_for_ratios([1.0] * 5) + for record in records: + if record["side"] != "candidate": + continue + for row in record["envelope"]["data"]["rows"]: + state = row["diagnostic_final_state"] + state["physical"]["container_count"] = 2 + state["physical"]["container_bytes"] = 2048 + state["physical_layout_sha256"] = "a" * 64 + stats = row["execution_stats"] + stats["container_append_count"] = 2 + stats["container_open_count"] = 2 + stats["container_close_count"] = 2 + stats["fsync_count"] = 2 + stats["io"].update( + {"container_opens": 2, "container_appends": 2, "fsyncs": 2, "bytes_written": 2048} + ) + recompute_top(record["envelope"]) + result = gate.compare_records( + records, + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(), + ) + self.assertEqual(result["classification"], "PASS") + + def test_canonical_content_mismatch_is_correctness_regression(self) -> None: + records = records_for_ratios([1.0] * 5) + for record in records: + if record["side"] == "candidate": + record["envelope"]["data"]["rows"][0]["diagnostic_final_state"]["physical"][ + "canonical_sha256" + ] = "a" * 64 + with self.assertRaises(gate.PairedGateError) as caught: + gate.compare_records( + records, + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(), + ) + self.assertEqual(caught.exception.classification, "CORRECTNESS_REGRESSION") + + def test_internal_hard_state_drift_is_evidence_failure(self) -> None: + records = records_for_ratios([1.0] * 5) + candidate = next(record for record in records[2:] if record["side"] == "candidate") + candidate["envelope"]["data"]["rows"][0]["diagnostic_final_state"][ + "active_logical_namespace" + ]["sha256"] = "a" * 64 + with self.assertRaises(gate.PairedGateError) as caught: + gate.compare_records( + records, + pair_count=5, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + mode="production", + thresholds=thresholds(), + ) + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + + def test_schema_unknown_field_and_fixture_mismatch_rejected(self) -> None: + report = raw_report() + report["data"]["new_field"] = 1 + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_raw_report(report, dataset="ci-paired-w1-v1", workers=1, compression="none") + self.assertEqual(caught.exception.classification, "CONTRACT_INVALID") + + report = raw_report() + report["data"]["fixture"]["mixed_file_count"] += 1 + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_raw_report(report, dataset="ci-paired-w1-v1", workers=1, compression="none") + self.assertEqual(caught.exception.classification, "EXECUTION_CONTRACT_MISMATCH") + + for invalid_duration in (0, -1, float("inf"), float("nan")): + with self.subTest(duration=invalid_duration): + report = raw_report() + report["data"]["rows"][0]["duration_ms"] = invalid_duration + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_raw_report( + report, + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + ) + self.assertEqual(caught.exception.classification, "CONTRACT_INVALID") + + def test_threshold_policy_is_complete_and_no_broader_than_ten_percent(self) -> None: + policy = { + "schema_version": 1, + "report_kind": gate.THRESHOLD_POLICY_KIND, + "contract_version": gate.CONTRACT_VERSION, + "policy_id": "test", + "cases": thresholds(10), + } + self.assertEqual(gate.validate_threshold_policy(policy), thresholds(10)) + policy["cases"][gate.PERFORMANCE_CASES[0]] = 10.001 + with self.assertRaises(gate.PairedGateError): + gate.validate_threshold_policy(policy) + + def test_reference_manifest_and_candidate_governance(self) -> None: + gate.validate_reference_manifest(manifest()) + changed = ["cmd/coldkeep/main.go", "benchmarks/paired/reference-v1.13.json"] + with self.assertRaises(gate.PairedGateError) as caught: + gate.reject_candidate_governance_changes(changed) + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + invalid = manifest() + invalid["reference_sha"] = "v1.13.11" + with self.assertRaises(gate.PairedGateError): + gate.validate_reference_manifest(invalid) + + def test_production_sampling_is_disabled_and_rejects_cli_reference(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + reference = root / "reference" + candidate = root / "candidate" + reference.write_bytes(b"same") + candidate.write_bytes(b"same") + base = [ + "sample", + "--reference-binary", + str(reference), + "--candidate-binary", + str(candidate), + "--candidate-sha", + "b" * 40, + "--dataset", + "ci-paired-w1-v1", + "--compression", + "none", + "--workers", + "1", + "--mode", + "production", + "--pairs", + "5", + "--go-version", + "go1.25.12", + "--postgres-version", + "16.14", + "--database-image-digest", + "sha256:" + "1" * 64, + ] + with mock.patch.dict(gate.os.environ, {"COLDKEEP_CODEC": "aes-gcm"}, clear=False): + output = root / "disabled" + exit_code = gate.main([*base, "--output-dir", str(output)]) + self.assertEqual(exit_code, 2) + report = gate.load_json_strict(output / "paired-comparison.json") + self.assertEqual(report["classification"], "REFERENCE_GOVERNANCE_INVALID") + self.assertEqual(report["governance_status"], "not-established") + + supplied = root / "supplied" + exit_code = gate.main( + [ + *base, + "--reference-sha", + "a" * 40, + "--output-dir", + str(supplied), + ] + ) + self.assertEqual(exit_code, 2) + report = gate.load_json_strict(supplied / "paired-comparison.json") + self.assertEqual(report["classification"], "REFERENCE_GOVERNANCE_INVALID") + + def test_reference_reachability_and_ancestry(self) -> None: + git = shutil.which("git") + if git is None: + self.skipTest("git is unavailable") + head = subprocess.run( + [git, "rev-parse", "HEAD"], check=True, text=True, capture_output=True + ).stdout.strip() + governed = manifest(head) + governed["approval"] = {"kind": "reviewed_record", "value": "phase11-test"} + gate.verify_reference_governance( + governed, + reference_sha=head, + candidate_sha=head, + repository=SCRIPT_DIR.parent, + ) + + def test_binary_identity_and_timeout_classifications(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + binary = root / "coldkeep" + binary.write_bytes(b"binary") + with self.assertRaises(gate.PairedGateError) as caught: + gate._capture( + binary=binary, + expected_hash="0" * 64, + side="candidate", + output_dir=root, + relative_raw_path=pathlib.Path("raw/candidate.json"), + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + timeout_seconds=1, + ) + self.assertEqual(caught.exception.classification, "BINARY_IDENTITY_INVALID") + + digest = gate._binary_hash(binary) + for side, expected in ( + ("candidate", "CANDIDATE_TIMEOUT_INCONCLUSIVE"), + ("reference", "CI_INFRASTRUCTURE_TIMEOUT"), + ): + with ( + mock.patch.object( + gate.subprocess, + "Popen", + return_value=FakeProcess(timeout=True), + ), + mock.patch.object(gate.os, "killpg"), + ): + with self.assertRaises(gate.PairedGateError) as caught: + gate._capture( + binary=binary, + expected_hash=digest, + side=side, + output_dir=root, + relative_raw_path=pathlib.Path(f"raw/{side}.json"), + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + timeout_seconds=1, + ) + self.assertEqual(caught.exception.classification, expected) + + def test_diagnostic_binary_mismatch_fails_before_sampling(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + reference = root / "reference" + candidate = root / "candidate" + reference.write_bytes(b"reference") + candidate.write_bytes(b"candidate") + output = root / "artifact" + argv = [ + "sample", + "--reference-binary", + str(reference), + "--candidate-binary", + str(candidate), + "--reference-sha", + "a" * 40, + "--candidate-sha", + "b" * 40, + "--output-dir", + str(output), + "--dataset", + "ci-paired-w1-v1", + "--compression", + "none", + "--workers", + "1", + "--mode", + "diagnostic", + "--pairs", + "10", + "--go-version", + "go1.25.12", + "--postgres-version", + "16.14", + "--database-image-digest", + "sha256:" + "1" * 64, + ] + with ( + mock.patch.dict(gate.os.environ, {"COLDKEEP_CODEC": "aes-gcm"}, clear=False), + mock.patch.object(gate, "_capture") as capture, + ): + self.assertEqual(gate.main(argv), 2) + capture.assert_not_called() + report = gate.load_json_strict(output / "paired-comparison.json") + self.assertEqual(report["classification"], "BINARY_IDENTITY_INVALID") + + def test_functional_failure_classification_is_side_specific(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + binary = root / "coldkeep" + binary.write_bytes(b"binary") + digest = gate._binary_hash(binary) + for side, expected in ( + ("candidate", "CANDIDATE_FUNCTIONAL_FAILURE"), + ("reference", "REFERENCE_FUNCTIONAL_FAILURE"), + ): + with mock.patch.object( + gate.subprocess, + "Popen", + return_value=FakeProcess(returncode=2, stderr="failed"), + ): + with self.assertRaises(gate.PairedGateError) as caught: + gate._capture( + binary=binary, + expected_hash=digest, + side=side, + output_dir=root, + relative_raw_path=pathlib.Path(f"raw/{side}.json"), + dataset="ci-paired-w1-v1", + workers=1, + compression="none", + timeout_seconds=1, + ) + self.assertEqual(caught.exception.classification, expected) + + def test_cli_failure_still_writes_immutable_profile_artifact(self) -> None: + with tempfile.TemporaryDirectory() as temp: + output = pathlib.Path(temp) / "artifact" + exit_code = gate.main( + [ + "sample", + "--reference-binary", + str(pathlib.Path(temp) / "missing-reference"), + "--candidate-binary", + str(pathlib.Path(temp) / "missing-candidate"), + "--reference-sha", + "a" * 40, + "--candidate-sha", + "b" * 40, + "--output-dir", + str(output), + "--dataset", + "ci-paired-w1-v2", + "--compression", + "none", + "--workers", + "1", + "--mode", + "diagnostic", + "--pairs", + "5", + "--go-version", + "go1.25.12", + "--postgres-version", + "16.14", + "--database-image-digest", + "sha256:" + "1" * 64, + ] + ) + self.assertEqual(exit_code, 2) + gate.validate_checksums(output) + report = gate.raw_gate.load_json_strict(output / "paired-comparison.json") + validated = gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(validated["classification"], "PAIR_INVENTORY_INVALID") + + def test_sigterm_and_manual_interrupt_write_non_performance_failures(self) -> None: + base = [ + "sample", + "--reference-binary", + "/nonexistent/reference", + "--candidate-binary", + "/nonexistent/candidate", + "--reference-sha", + "a" * 40, + "--candidate-sha", + "b" * 40, + "--dataset", + "ci-paired-w1-v1", + "--compression", + "none", + "--workers", + "1", + "--mode", + "diagnostic", + "--pairs", + "10", + "--go-version", + "go1.25.12", + "--postgres-version", + "16.14", + "--database-image-digest", + "sha256:" + "1" * 64, + ] + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + + def terminate(_args: argparse.Namespace) -> int: + gate.os.kill(gate.os.getpid(), gate.signal.SIGTERM) + return 0 + + terminated = root / "terminated" + with mock.patch.object(gate, "sample_command", side_effect=terminate): + self.assertEqual( + gate.main([*base, "--output-dir", str(terminated)]), 2 + ) + report = gate.load_json_strict(terminated / "paired-comparison.json") + self.assertEqual(report["classification"], "CI_INFRASTRUCTURE_TIMEOUT") + self.assertEqual(report["cleanup"]["status"], "complete") + + interrupted = root / "interrupted" + with mock.patch.object(gate, "sample_command", side_effect=KeyboardInterrupt): + self.assertEqual( + gate.main([*base, "--output-dir", str(interrupted)]), 2 + ) + report = gate.load_json_strict(interrupted / "paired-comparison.json") + self.assertEqual(report["classification"], "CI_INFRASTRUCTURE_TIMEOUT") + + def test_cleanup_and_report_unknown_field_fail_closed(self) -> None: + report = report_summary() + report["cleanup"]["failed"] = 1 + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + report = report_summary() + report["unknown"] = True + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "CONTRACT_INVALID") + + def test_complete_report_accepts_improvement_and_negative_delta(self) -> None: + report = report_summary() + for case in report["cases"]: + if not case["performance_gated"]: + continue + case["paired_ratios"] = [0.9] * 5 + case["median_ratio"] = 0.9 + case["regression_pct"] = -10.0 + gate.validate_report_summary(report, expected_profile="none-w1") + + def test_checksum_inventory_detects_tampering_and_missing_files(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + (root / "paired-comparison.json").write_text("{}\n", encoding="utf-8") + gate._write_checksums(root) + gate.validate_checksums(root) + (root / "paired-comparison.json").write_text("{\"changed\":true}\n", encoding="utf-8") + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_checksums(root) + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + + def test_strict_json_rejects_duplicate_keys_and_trailing_envelopes(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + duplicate = root / "duplicate.json" + duplicate.write_text('{"status":"ok","status":"pass"}\n', encoding="utf-8") + with self.assertRaises(gate.raw_gate.GateError): + gate.load_json_strict(duplicate) + trailing = root / "trailing.json" + trailing.write_text('{}\n{}\n', encoding="utf-8") + with self.assertRaises(gate.raw_gate.GateError): + gate.load_json_strict(trailing) + + def test_checksum_contract_rejects_symlinks_traversal_and_unexpected_files(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) / "artifact" + root.mkdir() + (root / "paired-comparison.json").write_text("{}\n", encoding="utf-8") + gate._write_checksums(root) + gate.validate_checksums(root, expected_files={"paired-comparison.json"}) + + (root / "unexpected.txt").write_text("unexpected\n", encoding="utf-8") + gate._write_checksums(root) + with self.assertRaises(gate.PairedGateError): + gate.validate_checksums(root, expected_files={"paired-comparison.json"}) + (root / "unexpected.txt").unlink() + + outside = pathlib.Path(temp) / "outside.txt" + outside.write_text("outside\n", encoding="utf-8") + (root / "linked.txt").symlink_to(outside) + with self.assertRaises(gate.PairedGateError): + gate._write_checksums(root) + (root / "linked.txt").unlink() + + digest = gate._binary_hash(root / "paired-comparison.json") + (root / "checksums.sha256").write_text( + f"{digest} ../paired-comparison.json\n", encoding="utf-8" + ) + with self.assertRaises(gate.PairedGateError): + gate.validate_checksums(root) + + report = report_summary() + report["invocation_inventory"][0]["raw_file"] = "../escape.json" + with self.assertRaises(gate.PairedGateError): + gate.validate_report_summary(report, expected_profile="none-w1") + + def test_governed_and_output_paths_reject_symlink_redirection(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + repository = root / "repository" + governed = repository / "benchmarks" / "paired" + governed.mkdir(parents=True) + outside = root / "outside.json" + outside.write_text("{}\n", encoding="utf-8") + (governed / "reference-v1.13.json").symlink_to(outside) + with self.assertRaises(gate.PairedGateError) as caught: + gate._governed_repository_file( + repository, + gate.GOVERNED_MANIFEST_RELATIVE, + "governed reference manifest", + ) + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + output_target = root / "output-target" + output_target.mkdir() + output_link = root / "output-link" + output_link.symlink_to(output_target, target_is_directory=True) + with self.assertRaises(gate.PairedGateError): + gate._create_output_directory(output_link) + + def test_captured_output_rejects_or_redacts_sensitive_values(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + raw = root / "raw" + raw.mkdir() + capture = raw / "failure.stderr" + capture.write_text( + "connection failed: postgresql://user:secret@localhost/private\n", + encoding="utf-8", + ) + with self.assertRaises(gate.PairedGateError) as caught: + gate._read_artifact_capture(capture, "test stderr") + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + + gate._sanitize_failure_captures(root) + self.assertEqual( + capture.read_text(encoding="utf-8"), + "[captured output omitted: sensitive content]\n", + ) + gate._read_artifact_capture(capture, "redacted stderr") + + def test_decision_requires_explicit_closed_mode(self) -> None: + with self.assertRaises(SystemExit): + gate.main( + [ + "decision", + "--profile", + "none-w1=/tmp/profile", + "--output-dir", + "/tmp/decision", + ] + ) + with self.assertRaises(SystemExit): + gate.main( + [ + "decision", + "--mode", + "unknown", + "--profile", + "none-w1=/tmp/profile", + "--output-dir", + "/tmp/decision", + ] + ) + + def test_diagnostic_decision_accepts_exact_matrix_and_is_non_authoritative(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + profiles = [] + for name in gate.PROFILE_MATRIX: + directory = root / name + write_complete_diagnostic_profile_artifact(directory, profile=name) + profiles.append(f"{name}={directory}") + output = root / "decision" + exit_code = gate.main( + [ + "decision", + "--mode", + "diagnostic", + *sum((["--profile", value] for value in profiles), []), + "--output-dir", + str(output), + ] + ) + self.assertEqual(exit_code, 0) + decision = gate.validate_decision_artifact( + output, expected_mode="diagnostic" + ) + self.assertEqual(decision["classification"], "DIAGNOSTIC_QUALIFIED") + self.assertEqual(decision["decision_scope"], "diagnostic_qualification") + self.assertEqual(decision["authority"], "diagnostic_only") + self.assertIs(decision["production_authority"], False) + self.assertNotEqual(decision["classification"], "PASS") + self.assertEqual(decision["matrix_coverage"], sorted(gate.PROFILE_MATRIX)) + self.assertEqual(decision["identity"]["reference_binary_sha256"], "c" * 64) + self.assertEqual( + decision["identity"]["reference_binary_sha256"], + decision["identity"]["candidate_binary_sha256"], + ) + self.assertTrue( + all(profile["raw_reconstruction"] == "verified" for profile in decision["profiles"]) + ) + + unknown = deepcopy(decision) + unknown["ignored"] = True + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_decision_report(unknown, expected_mode="diagnostic") + self.assertEqual(caught.exception.classification, "CONTRACT_INVALID") + + authoritative = deepcopy(decision) + authoritative["production_authority"] = True + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_decision_report(authoritative, expected_mode="diagnostic") + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + def test_mode_isolation_and_production_decisions_remain_disabled(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + diagnostic = root / "diagnostic" + write_summary_artifact( + diagnostic, report_summary(mode="diagnostic") + ) + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_profile_artifact( + diagnostic, expected_profile="none-w1", expected_mode="production" + ) + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + production = root / "production" + write_summary_artifact(production, report_summary()) + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_profile_artifact( + production, expected_profile="none-w1", expected_mode="diagnostic" + ) + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + args = argparse.Namespace( + mode="production", + profile=[f"{name}={root / name}" for name in gate.PROFILE_MATRIX], + output_dir=root / "decision", + ) + with self.assertRaises(gate.PairedGateError) as caught: + gate.decision_command(args) + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + mixed_profiles = [] + for name in gate.PROFILE_MATRIX: + directory = root / f"mixed-{name}" + write_summary_artifact( + directory, + report_summary( + profile=name, + mode="production" if name == "none-w1" else "diagnostic", + ), + ) + mixed_profiles.append(f"{name}={directory}") + with self.assertRaises(gate.PairedGateError) as caught: + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=mixed_profiles, + output_dir=root / "mixed-decision", + ) + ) + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + def test_sample_and_decision_require_new_harness_owned_output_directories(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + for name in ("sample", "decision"): + output = root / name + output.mkdir() + with self.subTest(command=name): + with self.assertRaises(gate.PairedGateError) as caught: + if name == "sample": + gate._create_output_directory(output) + else: + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[ + f"{profile}={root / profile}" + for profile in gate.PROFILE_MATRIX + ], + output_dir=output, + ) + ) + self.assertEqual( + caught.exception.classification, + "EVIDENCE_INTEGRITY_FAILURE", + ) + self.assertEqual(str(caught.exception), "output directory already exists") + + decision_output = root / "new-decision" + args = argparse.Namespace( + mode="diagnostic", + profile=[ + f"{profile}={root / profile}" + for profile in gate.PROFILE_MATRIX + ], + output_dir=decision_output, + ) + with self.assertRaises(gate.PairedGateError): + gate.decision_command(args) + self.assertTrue(decision_output.is_dir()) + self.assertTrue(args._output_owned) + + def test_diagnostic_artifact_binary_identity_and_authority_are_strict(self) -> None: + report = report_summary(mode="diagnostic") + report["identity"]["candidate_binary_sha256"] = "d" * 64 + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "BINARY_IDENTITY_INVALID") + + report = report_summary(mode="diagnostic") + report["authority"]["production_authority"] = True + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_report_summary(report, expected_profile="none-w1") + self.assertEqual(caught.exception.classification, "REFERENCE_GOVERNANCE_INVALID") + + def test_complete_artifact_is_recomputed_from_raw_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + repository = root / "repository" + repository.mkdir() + write_governed_repository(repository) + artifact = root / "artifact" + write_complete_profile_artifact(artifact, repository) + with ( + mock.patch.object(gate, "PRODUCTION_SAMPLING_AUTHORIZED", True), + mock.patch.object(gate, "_repository_root", return_value=repository), + mock.patch.object(gate, "verify_reference_governance"), + ): + gate.validate_profile_artifact( + artifact, expected_profile="none-w1", expected_mode="production" + ) + + raw_path = artifact / "raw" / "pair-01" / "02-candidate.json" + raw = gate.load_json_strict(raw_path) + row = raw["data"]["rows"][0] + row["duration_ms"] = 1100 + row["throughput_mbps"] = 1024 / (1024 * 1024) / 1.1 + gate.raw_gate.write_json(raw_path, raw) + gate._write_checksums(artifact) + with self.assertRaises(gate.PairedGateError) as caught: + gate.validate_profile_artifact( + artifact, expected_profile="none-w1", expected_mode="production" + ) + self.assertEqual(caught.exception.classification, "EVIDENCE_INTEGRITY_FAILURE") + + def test_decision_rejects_missing_duplicate_and_cross_profile_identity(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + with self.assertRaises(gate.PairedGateError): + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[f"none-w1={root}"], + output_dir=root / "missing-out", + ) + ) + with self.assertRaises(gate.PairedGateError): + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[f"none-w1={root}", f"none-w1={root}"], + output_dir=root / "duplicate-out", + ) + ) + with self.assertRaises(gate.PairedGateError): + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[ + *[f"{name}={root / name}" for name in gate.PROFILE_MATRIX], + f"extra={root / 'extra'}", + ], + output_dir=root / "extra-out", + ) + ) + + reports = [ + report_summary(profile=name, mode="diagnostic") + for name in sorted(gate.PROFILE_MATRIX) + ] + reports[1]["identity"]["candidate_sha"] = "c" * 40 + with mock.patch.object(gate, "validate_profile_artifact", side_effect=reports): + with self.assertRaises(gate.PairedGateError) as caught: + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[f"{name}={root / name}" for name in sorted(gate.PROFILE_MATRIX)], + output_dir=root / "identity-out", + ) + ) + self.assertEqual(caught.exception.classification, "EXECUTION_CONTRACT_MISMATCH") + + for field, value in ( + ("manifest_sha256", "1" * 64), + ("threshold_policy_id", "different-policy"), + ("threshold_policy_sha256", "2" * 64), + ): + with self.subTest(governance_field=field): + reports = [ + report_summary(profile=name, mode="diagnostic") + for name in sorted(gate.PROFILE_MATRIX) + ] + reports[3]["governance"][field] = value + with mock.patch.object( + gate, "validate_profile_artifact", side_effect=reports + ): + with self.assertRaises(gate.PairedGateError) as caught: + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[ + f"{name}={root / name}" + for name in sorted(gate.PROFILE_MATRIX) + ], + output_dir=root / f"governance-{field}-out", + ) + ) + self.assertEqual( + caught.exception.classification, + "EXECUTION_CONTRACT_MISMATCH", + ) + + reports = [ + report_summary(profile=name, mode="diagnostic") + for name in sorted(gate.PROFILE_MATRIX) + ] + reports[2]["identity"]["reference_binary_sha256"] = "0" * 64 + reports[2]["identity"]["candidate_binary_sha256"] = "0" * 64 + with mock.patch.object(gate, "validate_profile_artifact", side_effect=reports): + with self.assertRaises(gate.PairedGateError) as caught: + gate.decision_command( + argparse.Namespace( + mode="diagnostic", + profile=[ + f"{name}={root / name}" + for name in sorted(gate.PROFILE_MATRIX) + ], + output_dir=root / "binary-out", + ) + ) + self.assertEqual(caught.exception.classification, "BINARY_IDENTITY_INVALID") + + def test_decision_precedence_and_matrix_coverage(self) -> None: + self.assertEqual( + gate.decision_classification( + ["PASS", "PERFORMANCE_REGRESSION", "CORRECTNESS_REGRESSION"], + mode="production", + ), + "CORRECTNESS_REGRESSION", + ) + self.assertEqual( + gate.decision_classification( + ["PERFORMANCE_REGRESSION", "CANDIDATE_TIMEOUT_INCONCLUSIVE"], + mode="production", + ), + "CANDIDATE_TIMEOUT_INCONCLUSIVE", + ) + self.assertEqual( + gate.decision_classification( + ["DIAGNOSTIC_QUALIFIED"] * 4, mode="diagnostic" + ), + "DIAGNOSTIC_QUALIFIED", + ) + for profile in gate.PROFILE_MATRIX: + gate.validate_report_summary(report_summary(profile=profile), expected_profile=profile) + + def test_internal_profile_deadline_boundary_and_immediate_exhaustion(self) -> None: + with mock.patch.object(gate.time, "monotonic", return_value=2100.0): + with self.assertRaises(gate.PairedGateError) as caught: + gate._profile_elapsed_ms_or_fail(0.0, "diagnostic") + self.assertEqual(caught.exception.classification, "DIAGNOSTIC_TIME_BUDGET_EXCEEDED") + + with mock.patch.object(gate.time, "monotonic", return_value=2099.999): + self.assertEqual(gate._profile_elapsed_ms_or_fail(0.0, "diagnostic"), 2099999.0) + + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + binary = root / "coldkeep" + binary.write_bytes(b"binary") + with ( + mock.patch.object(gate.time, "monotonic", return_value=100.0), + mock.patch.object(gate.subprocess, "Popen") as popen, + ): + with self.assertRaises(gate.PairedGateError) as caught: + gate._capture( + binary=binary, + expected_hash=gate._binary_hash(binary), + side="candidate", + output_dir=root, + relative_raw_path=pathlib.Path("raw/candidate.json"), + dataset="ci-paired-w1-v2", + workers=1, + compression="none", + timeout_seconds=600, + profile_deadline=100.0, + ) + self.assertEqual(caught.exception.classification, "DIAGNOSTIC_TIME_BUDGET_EXCEEDED") + popen.assert_not_called() + + process = FakeProcess(timeout=True) + state = {"active_invocation": None, "process_started": False} + with ( + mock.patch.object(gate.time, "monotonic", side_effect=[100.0, 100.0]), + mock.patch.object(gate.subprocess, "Popen", return_value=process), + mock.patch.object(gate.os, "killpg"), + ): + with self.assertRaises(gate.PairedGateError) as caught: + gate._capture( + binary=binary, + expected_hash=gate._binary_hash(binary), + side="candidate", + output_dir=root, + relative_raw_path=pathlib.Path("raw/deadline-candidate.json"), + dataset="ci-paired-w1-v2", + workers=1, + compression="none", + timeout_seconds=600, + profile_deadline=105.0, + profile_state=state, + invocation={ + "kind": "warmup", + "pair_ordinal": None, + "position": 1, + "side": "candidate", + }, + ) + self.assertEqual(caught.exception.classification, "DIAGNOSTIC_TIME_BUDGET_EXCEEDED") + self.assertEqual(process.timeout_values[0], 5.0) + + def test_owned_process_group_escalates_and_reaps(self) -> None: + process = FakeProcess(timeout=True) + calls = 0 + + def stubborn(timeout: float | None = None) -> tuple[str, str]: + nonlocal calls + calls += 1 + if calls <= 1: + raise subprocess.TimeoutExpired(["coldkeep"], timeout or 0) + process.returncode = 0 + return "", "" + + process.communicate = stubborn # type: ignore[method-assign] + with mock.patch.object(gate.os, "killpg") as killpg: + gate._terminate_process_group(process) + self.assertEqual( + killpg.call_args_list, + [ + mock.call(process.pid, gate.signal.SIGTERM), + mock.call(process.pid, gate.signal.SIGKILL), + ], + ) + self.assertEqual(calls, 2) + + def test_interrupted_profile_artifact_contains_only_validated_prefix(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + output = root / "artifact" + output.mkdir() + reference = root / "reference" + candidate = root / "candidate" + reference.write_bytes(b"same") + candidate.write_bytes(b"same") + raw_path = output / "raw" / "warmup-01-candidate.json" + raw_path.parent.mkdir(parents=True) + gate.raw_gate.write_json( + raw_path, + raw_report(dataset="ci-paired-w1-v2", workers=1), + ) + raw_path.with_suffix(".stderr").write_text("", encoding="utf-8") + active_raw = output / "raw" / "warmup-02-reference.json" + active_raw.write_text('{"status":', encoding="utf-8") + active_raw.with_suffix(".stderr").write_text( + "terminated before JSON completion\n", encoding="utf-8" + ) + args = argparse.Namespace( + output_dir=output, + mode="diagnostic", + reference_sha="a" * 40, + candidate_sha="b" * 40, + reference_binary=reference, + candidate_binary=candidate, + compression="none", + dataset="ci-paired-w1-v2", + workers=1, + pairs=10, + go_version="go1.25.12", + postgres_version="16.14", + database_image_digest="sha256:" + "1" * 64, + ) + args._profile_state = { + "started": 1.0, + "active_invocation": { + "kind": "warmup", + "pair_ordinal": None, + "position": 2, + "side": "reference", + }, + "cancellation_reason": "internal profile deadline reached", + } + with ( + mock.patch.object(gate.time, "monotonic", return_value=2.0), + mock.patch.object( + gate, + "_cleanup_interrupted_profile", + return_value={ + "status": "complete", + "filesystem_entries_removed": 1, + "databases_removed": 1, + "errors": 0, + }, + ), + ): + gate._write_failure_artifact( + args, + gate.PairedGateError( + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", "deadline" + ), + ) + report = gate.validate_profile_artifact( + output, expected_profile="none-w1", expected_mode="diagnostic" + ) + self.assertEqual(report["classification"], "DIAGNOSTIC_TIME_BUDGET_EXCEEDED") + self.assertEqual(report["prefix_validation"]["raw_report_count"], 1) + self.assertEqual(report["active_invocation"]["status"], "incomplete") + self.assertEqual( + report["active_invocation"]["capture_validation"], "unvalidated" + ) + self.assertTrue(report["active_invocation"]["raw_capture_present"]) + self.assertNotIn("cases", report) + self.assertNotIn("paired_ratios", str(report)) + + failed_output = root / "cleanup-failed" + failed_output.mkdir() + failed_args = argparse.Namespace(**vars(args)) + failed_args.output_dir = failed_output + failed_args._profile_state = { + **args._profile_state, + "active_invocation": { + "kind": "warmup", + "pair_ordinal": None, + "position": 1, + "side": "candidate", + }, + } + with ( + mock.patch.object(gate.time, "monotonic", return_value=2.0), + mock.patch.object( + gate, + "_cleanup_interrupted_profile", + return_value={ + "status": "incomplete", + "filesystem_entries_removed": 0, + "databases_removed": 0, + "errors": 1, + }, + ), + ): + classification = gate._write_failure_artifact( + failed_args, + gate.PairedGateError( + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", "deadline" + ), + ) + self.assertEqual(classification, "EVIDENCE_INTEGRITY_FAILURE") + failed_report = gate.validate_profile_artifact( + failed_output, + expected_profile="none-w1", + expected_mode="diagnostic", + ) + self.assertEqual(failed_report["cleanup"]["status"], "incomplete") + self.assertEqual( + failed_report["classification"], "EVIDENCE_INTEGRITY_FAILURE" + ) + + def test_decision_failures_are_owned_checksummed_and_non_authoritative(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + output = root / "missing-decision" + argv = ["decision", "--mode", "diagnostic"] + for name in gate.PROFILE_MATRIX: + argv.extend(["--profile", f"{name}={root / name}"]) + argv.extend(["--output-dir", str(output)]) + self.assertEqual(gate.main(argv), 2) + report = gate.validate_decision_artifact(output, expected_mode="diagnostic") + self.assertEqual(report["status"], "failed") + self.assertEqual(report["classification"], "EVIDENCE_INTEGRITY_FAILURE") + self.assertTrue(all(value is None for value in report["identity"].values())) + self.assertEqual(report["evidence"]["hard_state"]["status"], "not_verified") + + incomplete = root / "incomplete-decision" + incomplete_argv = ["decision", "--mode", "diagnostic"] + for name in gate.PROFILE_MATRIX: + incomplete_argv.extend(["--profile", f"{name}={root / name}"]) + incomplete_argv.extend(["--output-dir", str(incomplete)]) + failed_profile = { + "status": "failed", + "classification": "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + } + with mock.patch.object( + gate, "validate_profile_artifact", return_value=failed_profile + ): + self.assertEqual(gate.main(incomplete_argv), 2) + report = gate.validate_decision_artifact(incomplete, expected_mode="diagnostic") + self.assertEqual(report["classification"], "DIAGNOSTIC_TIME_BUDGET_EXCEEDED") + self.assertEqual(report["profiles"][0]["status"], "failed") + + def decision_argv(label: str) -> list[str]: + values = ["decision", "--mode", "diagnostic"] + for profile_name in gate.PROFILE_MATRIX: + values.extend( + ["--profile", f"{profile_name}={root / profile_name}"] + ) + values.extend(["--output-dir", str(root / label)]) + return values + + failure_cases = ( + ( + "functional-decision", + { + "status": "failed", + "classification": "REFERENCE_FUNCTIONAL_FAILURE", + }, + "REFERENCE_FUNCTIONAL_FAILURE", + ), + ( + "tampered-decision", + gate.PairedGateError( + "EVIDENCE_INTEGRITY_FAILURE", "checksum mismatch" + ), + "EVIDENCE_INTEGRITY_FAILURE", + ), + ( + "malformed-decision", + gate.PairedGateError("CONTRACT_INVALID", "malformed raw"), + "CONTRACT_INVALID", + ), + ) + for label, validator_result, expected_classification in failure_cases: + with self.subTest(decision_failure=label): + with mock.patch.object( + gate, + "validate_profile_artifact", + side_effect=validator_result + if isinstance(validator_result, BaseException) + else None, + return_value=validator_result + if isinstance(validator_result, dict) + else mock.DEFAULT, + ): + self.assertEqual(gate.main(decision_argv(label)), 2) + report = gate.validate_decision_artifact( + root / label, expected_mode="diagnostic" + ) + self.assertEqual( + report["classification"], expected_classification + ) + + mixed_source_reports = [ + report_summary(profile=name, mode="diagnostic") + for name in sorted(gate.PROFILE_MATRIX) + ] + mixed_source_reports[1]["identity"]["candidate_sha"] = "c" * 40 + with mock.patch.object( + gate, + "validate_profile_artifact", + side_effect=mixed_source_reports, + ): + self.assertEqual(gate.main(decision_argv("mixed-source-decision")), 2) + report = gate.validate_decision_artifact( + root / "mixed-source-decision", expected_mode="diagnostic" + ) + self.assertEqual(report["classification"], "EXECUTION_CONTRACT_MISMATCH") + + mixed_binary_reports = [ + report_summary(profile=name, mode="diagnostic") + for name in sorted(gate.PROFILE_MATRIX) + ] + mixed_binary_reports[2]["identity"]["reference_binary_sha256"] = "0" * 64 + mixed_binary_reports[2]["identity"]["candidate_binary_sha256"] = "0" * 64 + with mock.patch.object( + gate, + "validate_profile_artifact", + side_effect=mixed_binary_reports, + ): + self.assertEqual(gate.main(decision_argv("mixed-binary-decision")), 2) + report = gate.validate_decision_artifact( + root / "mixed-binary-decision", expected_mode="diagnostic" + ) + self.assertEqual(report["classification"], "BINARY_IDENTITY_INVALID") + + for name in gate.PROFILE_MATRIX: + (root / name).mkdir() + interrupted = root / "interrupted-decision" + interrupted_argv = ["decision", "--mode", "diagnostic"] + for name in gate.PROFILE_MATRIX: + interrupted_argv.extend(["--profile", f"{name}={root / name}"]) + interrupted_argv.extend(["--output-dir", str(interrupted)]) + with mock.patch.object( + gate, "validate_profile_artifact", side_effect=KeyboardInterrupt + ): + self.assertEqual(gate.main(interrupted_argv), 2) + report = gate.validate_decision_artifact( + interrupted, expected_mode="diagnostic" + ) + self.assertEqual(report["classification"], "CI_INFRASTRUCTURE_TIMEOUT") + self.assertTrue( + all(item["raw_reconstruction"] != "verified" for item in report["profiles"]) + ) + + self.assertEqual( + gate.decision_classification( + ["DIAGNOSTIC_TIME_BUDGET_EXCEEDED", "DIAGNOSTIC_REJECTED"], + mode="diagnostic", + ), + "DIAGNOSTIC_TIME_BUDGET_EXCEEDED", + ) + + def test_historical_absolute_fixture_has_no_paired_authority(self) -> None: + self.assertNotIn("ci-stable-v1", gate.FIXTURES) + self.assertNotIn("ci-stable-v1", {profile[2] for profile in gate.PROFILE_MATRIX.values()}) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_validate_regression_thresholds.py b/scripts/test_validate_regression_thresholds.py new file mode 100644 index 00000000..fa05be55 --- /dev/null +++ b/scripts/test_validate_regression_thresholds.py @@ -0,0 +1,308 @@ +import contextlib +import copy +import io +import json +import pathlib +import sys +import tempfile +import unittest +from unittest import mock + +from scripts import validate_regression_thresholds as validator +from scripts.test_benchmark_gate import raw_report + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +BASELINE = ( + ROOT + / "benchmarks/v1.9/baselines/benchmark-baseline-v1.9-packed-aes-gcm-zstd-small-w1-r1.json" +) +THRESHOLDS = ROOT / "benchmarks/v1.9/regression-thresholds.yaml" + + +def candidate_from_baseline() -> dict: + candidate = copy.deepcopy(json.loads(BASELINE.read_text(encoding="utf-8"))) + data = candidate["data"] + data["schema_version"] = 2 + data["fixture"] = { + **validator.SMALL_FIXTURE, + "ordered_cases": [ + {"name": name, "seed": 1712 + index * 10} + for index, name in enumerate(validator.EXPECTED_CASES) + ], + } + return candidate + + +def valid_diagnostic_final_state() -> dict: + return raw_report(workers=1, dataset="ci-paired-w1-v2")["data"]["rows"][0][ + "diagnostic_final_state" + ] + + +class AdvisoryComparatorTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temp.name) + self.candidate = self.root / "candidate.json" + self.report = self.root / "timing-advisory.json" + + def tearDown(self) -> None: + self.temp.cleanup() + + def invoke(self, *arguments: str) -> int: + with mock.patch.object(sys, "argv", ["validate_regression_thresholds.py", *arguments]): + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + return validator.main() + + def write_candidate(self, candidate: dict) -> None: + self.candidate.write_text(json.dumps(candidate) + "\n", encoding="utf-8") + + def check(self) -> int: + return self.invoke( + "check", + str(self.candidate), + "--baseline", + str(BASELINE), + "--mode", + "compressed", + "--thresholds", + str(THRESHOLDS), + "--policy", + "hosted-advisory", + "--json-report", + str(self.report), + ) + + def test_within_reference_exit_and_report_verify_exactly(self) -> None: + self.write_candidate(candidate_from_baseline()) + self.assertEqual(self.check(), 0) + report = json.loads(self.report.read_text(encoding="utf-8")) + self.assertEqual(report["classification"], "BENCHMARK_TIMING_WITHIN_REFERENCE") + self.assertEqual(report["reference_kind"], "historical_v1.9_absolute") + self.assertEqual( + self.invoke( + "verify-advisory-exit", + "--report", + str(self.report), + "--observed-exit-code", + "0", + ), + 0, + ) + self.assertEqual( + self.invoke( + "verify-advisory-exit", + "--report", + str(self.report), + "--observed-exit-code", + "10", + ), + 2, + ) + + def test_real_shape_small_observation_does_not_require_diagnostic_state(self) -> None: + candidate = candidate_from_baseline() + self.assertTrue(all("diagnostic_final_state" not in row for row in candidate["data"]["rows"])) + self.write_candidate(candidate) + self.assertEqual(self.check(), 0) + + def test_go_omitempty_operational_zeroes_may_all_be_absent(self) -> None: + candidate = candidate_from_baseline() + for stats in [ + candidate["data"]["execution_stats"], + *(row["execution_stats"] for row in candidate["data"]["rows"]), + ]: + for field in validator.EXECUTION_STATS_OMITTABLE_ZERO_FIELDS: + stats.pop(field, None) + stats["io"]["container_opens"] = 0 + stats["io"]["container_appends"] = 0 + stats["io"]["fsyncs"] = 0 + stats["io"]["bytes_written"] = 0 + stats["io"]["bytes_read"] = 0 + self.write_candidate(candidate) + self.assertEqual(self.check(), 0) + + def test_optional_diagnostic_final_state_is_structurally_validated(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["rows"][0]["diagnostic_final_state"] = valid_diagnostic_final_state() + self.write_candidate(candidate) + self.assertEqual(self.check(), 0) + + candidate["data"]["rows"][0]["diagnostic_final_state"] = {"schema_version": 2} + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + self.assert_failure_report() + + def test_warning_exit_is_ten_and_never_pass(self) -> None: + candidate = candidate_from_baseline() + row = candidate["data"]["rows"][0] + row["duration_ms"] *= 2 + row["throughput_mbps"] = ( + row["execution_stats"]["total_bytes"] + / (1024 * 1024) + / (row["duration_ms"] / 1000.0) + ) + self.write_candidate(candidate) + self.assertEqual(self.check(), 10) + report = json.loads(self.report.read_text(encoding="utf-8")) + self.assertEqual(report["classification"], "BENCHMARK_TIMING_WARNING") + self.assertGreater(report["violations_count"], 0) + self.assertNotIn("passed", report) + + def test_unknown_candidate_data_is_error_with_failure_report(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["unknown"] = True + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + self.assert_failure_report() + + def test_unknown_row_and_execution_stat_fields_fail_closed(self) -> None: + for location in ("row", "stats"): + with self.subTest(location=location): + candidate = candidate_from_baseline() + target = candidate["data"]["rows"][0] + if location == "stats": + target = target["execution_stats"] + target["unknown"] = True + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + self.assert_failure_report() + + def test_fixture_order_and_seed_mismatches_are_errors(self) -> None: + mutations = ( + lambda candidate: candidate["data"]["fixture"].__setitem__("seed", 1702), + lambda candidate: candidate["data"]["rows"].reverse(), + lambda candidate: candidate["data"]["fixture"]["ordered_cases"][0].__setitem__("seed", 1713), + ) + for mutation in mutations: + with self.subTest(mutation=mutation): + candidate = candidate_from_baseline() + mutation(candidate) + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + self.assert_failure_report() + + def test_counter_total_mismatch_is_error(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["execution_stats"]["total_files"] += 1 + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + self.assert_failure_report() + + def test_duplicated_io_counter_mismatch_is_error(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["rows"][0]["execution_stats"]["io"]["container_opens"] += 1 + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + + def test_derived_throughput_mismatch_is_error(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["rows"][0]["throughput_mbps"] += 0.1 + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + + def assert_failure_report(self) -> dict: + report = json.loads(self.report.read_text(encoding="utf-8")) + self.assertEqual(report["status"], "failed") + self.assertEqual(report["classification"], "BENCHMARK_TIMING_EVALUATION_FAILURE") + self.assertEqual(report["authority"], "informational") + self.assertEqual(report["violations_count"], 0) + self.assertEqual(report["violations"], []) + self.assertNotIn("passed", report) + return report + + def test_missing_and_malformed_candidate_emit_evaluator_failure(self) -> None: + self.assertEqual(self.check(), 2) + missing = self.assert_failure_report() + self.assertEqual(missing["error"]["category"], "missing_input") + self.assertNotIn("candidate_sha256", missing) + + self.candidate.write_text("{malformed\n", encoding="utf-8") + self.assertEqual(self.check(), 2) + malformed = self.assert_failure_report() + self.assertEqual(malformed["error"]["category"], "malformed_input") + self.assertIn("candidate_sha256", malformed) + + def test_evaluator_failure_report_exit_two_verifies_exactly(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["unknown"] = True + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + self.assertEqual(validator.verify_advisory_exit(self.report, 2), 0) + with self.assertRaises(RuntimeError): + validator.verify_advisory_exit(self.report, 12) + + def test_evaluator_failure_cannot_claim_not_evaluated_timing_or_production_authority(self) -> None: + candidate = candidate_from_baseline() + candidate["data"]["unknown"] = True + self.write_candidate(candidate) + self.assertEqual(self.check(), 2) + base = self.assert_failure_report() + mutations = ( + {"classification": "BENCHMARK_TIMING_NOT_EVALUATED"}, + {"timing_within_reference": True}, + {"performance_authority": True}, + {"authority": "production"}, + ) + for mutation in mutations: + with self.subTest(mutation=mutation): + report = {**base, **mutation} + self.report.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaises(RuntimeError): + validator.verify_advisory_exit(self.report, 2) + + def test_all_declared_advisory_exit_classifications_verify(self) -> None: + base = { + "schema_version": 1, + "report_kind": validator.ADVISORY_REPORT_KIND, + "status": "complete", + "classification": "", + "authority": "informational", + "reference_kind": "historical_v1.9_absolute", + "mode": "compressed", + "candidate_sha256": "a" * 64, + "baseline_sha256": "b" * 64, + "violations_count": 0, + "violations": [], + } + for classification, exit_code in validator.ADVISORY_EXIT_CODES.items(): + with self.subTest(classification=classification): + report = copy.deepcopy(base) + report["classification"] = classification + if classification == "BENCHMARK_TIMING_WARNING": + report["violations"] = [{"case": "store-large-file"}] + report["violations_count"] = 1 + if classification == "BENCHMARK_TIMING_NOT_EVALUATED": + report["status"] = "not_evaluated" + if classification == "BENCHMARK_TIMING_EVALUATION_FAILURE": + report["status"] = "failed" + report["error"] = { + "category": "contract_error", + "message": "benchmark advisory evidence violates its contract", + } + self.report.write_text(json.dumps(report), encoding="utf-8") + self.assertEqual(validator.verify_advisory_exit(self.report, exit_code), 0) + + def test_legacy_default_keeps_exit_one_for_hard_regression(self) -> None: + candidate = json.loads(BASELINE.read_text(encoding="utf-8")) + row = candidate["data"]["rows"][0] + row["duration_ms"] *= 2 + self.write_candidate(candidate) + self.assertEqual( + self.invoke( + "check", + str(self.candidate), + "--baseline", + str(BASELINE), + "--mode", + "uncompressed", + "--thresholds", + str(THRESHOLDS), + ), + 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_regression_thresholds.py b/scripts/validate_regression_thresholds.py index 97b68b11..3276530d 100755 --- a/scripts/validate_regression_thresholds.py +++ b/scripts/validate_regression_thresholds.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -""" -Coldkeep Benchmark Regression Threshold Validator +"""Coldkeep legacy single-observation benchmark threshold validator. Validates threshold configuration realism and applies thresholds to benchmark runs. Supports mode-specific (uncompressed/compressed) and case-specific thresholds. +The required aggregate v2 release gate is owned by scripts/benchmark_gate.py. Usage: # Validate that thresholds are realistic against baselines @@ -20,12 +20,39 @@ """ import argparse +import hashlib import json +import math import pathlib import sys from typing import Any, Dict, List, Optional, Tuple import yaml +try: + from scripts import benchmark_gate as benchmark_contract +except ImportError: # Direct execution from scripts/ places that directory on sys.path. + import benchmark_gate as benchmark_contract + +ADVISORY_REPORT_KIND = "benchmark_timing_advisory" +ADVISORY_EXIT_CODES = { + "BENCHMARK_TIMING_WITHIN_REFERENCE": 0, + "BENCHMARK_TIMING_WARNING": 10, + "BENCHMARK_TIMING_UNSTABLE": 11, + "BENCHMARK_TIMING_NOT_EVALUATED": 12, + "BENCHMARK_TIMING_EVALUATION_FAILURE": 2, +} +ADVISORY_FAILURE_CATEGORIES = { + "configuration_error", "contract_error", "io_error", "malformed_input", "missing_input", +} + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + def load_yaml(path: pathlib.Path) -> Dict[str, Any]: """Load YAML configuration file.""" @@ -43,6 +70,10 @@ def load_benchmark_envelope(path: pathlib.Path) -> Dict[str, Any]: obj = json.loads(line) except json.JSONDecodeError: continue + if isinstance(obj, dict) and obj.get('report_kind') == 'benchmark_gate_aggregate': + raise RuntimeError( + f"Aggregate v2 evidence must be validated with scripts/benchmark_gate.py: {path}" + ) if isinstance(obj, dict) and 'data' in obj: return obj raise RuntimeError(f"No valid benchmark envelope found in {path}") @@ -64,6 +95,269 @@ def extract_rows(envelope: Dict[str, Any]) -> Dict[str, Dict[str, float]]: return rows +EXPECTED_CASES = [ + "store-large-file", "store-many-small-files", "store-mixed-dataset", + "restore-large-file", "restore-many-files", "snapshot-creation", + "gc-after-churn", "stats-inspect", "verify-system-deep", +] +SMALL_FIXTURE = { + "id": "small", "seed": 1701, "large_file_size_bytes": 16 * 1024 * 1024, + "many_small_file_count": 100, "many_small_file_size_bytes": 1024, + "mixed_file_count": 20, "mixed_min_file_size_bytes": 1024, + "mixed_max_file_size_bytes": 256 * 1024, "remove_every": 4, + "case_database_isolation": False, +} +TIMING_ROW_REQUIRED_FIELDS = { + "case", "duration_ms", "throughput_mbps", "execution", "execution_stats", +} +# The ordinary historical `small` path does not install the diagnostic observer. +# Integrity-only hard state remains required by benchmark_gate.py, not this timing contract. +TIMING_ROW_OPTIONAL_FIELDS = {"diagnostic_final_state"} +EXECUTION_STATS_REQUIRED_FIELDS = {"total_files", "total_bytes", "workers_used", "io"} +# These are the only BenchmarkExecutionStats fields shipped with Go `omitempty` tags. +EXECUTION_STATS_OMITTABLE_ZERO_FIELDS = { + "container_append_count", "fsync_count", "container_open_count", + "container_close_count", "snapshot_metadata_write_count", +} + + +def require_fields(value: Any, expected: set[str], label: str) -> Dict[str, Any]: + if not isinstance(value, dict) or set(value) != expected: + raise RuntimeError(f"{label} fields mismatch") + return value + + +def validate_execution(value: Any, workers: int, label: str) -> None: + value = require_fields( + value, {"store_folder_workers", "pipeline_depth", "deterministic"}, label + ) + if value != { + "store_folder_workers": workers, "pipeline_depth": 1, "deterministic": True + }: + raise RuntimeError(f"{label} policy mismatch") + + +def validate_execution_stats( + value: Any, *, workers: int, label: str, case_name: Optional[str] = None, +) -> Dict[str, Any]: + normalized, io = normalize_execution_stats(value, label) + validate_execution_stat_counters(normalized, io, label) + validate_execution_stat_totals(normalized, io, workers, label) + validate_execution_stat_io_contract(normalized, io, label) + validate_execution_stat_snapshot(normalized, case_name, label) + return normalized + + +def normalize_execution_stats(value: Any, label: str) -> Tuple[Dict[str, Any], Dict[str, Any]]: + if ( + not isinstance(value, dict) + or not EXECUTION_STATS_REQUIRED_FIELDS <= set(value) + or set(value) - EXECUTION_STATS_REQUIRED_FIELDS - EXECUTION_STATS_OMITTABLE_ZERO_FIELDS + ): + raise RuntimeError(f"{label} fields mismatch") + io = require_fields( + value["io"], + {"container_opens", "container_appends", "fsyncs", "bytes_written", "bytes_read"}, + f"{label} I/O", + ) + normalized = { + **value, + **{field: value.get(field, 0) for field in EXECUTION_STATS_OMITTABLE_ZERO_FIELDS}, + } + return normalized, io + + +def validate_execution_stat_counters( + normalized: Dict[str, Any], io: Dict[str, Any], label: str, +) -> None: + counters = [ + *(normalized[field] for field in EXECUTION_STATS_REQUIRED_FIELDS - {"io"}), + *(normalized[field] for field in EXECUTION_STATS_OMITTABLE_ZERO_FIELDS), + *io.values(), + ] + for item in counters: + try: + benchmark_contract.require_nonnegative_integer(item, label) + except benchmark_contract.GateError as exc: + raise RuntimeError(f"{label} counter invalid") from exc + + +def validate_execution_stat_totals( + normalized: Dict[str, Any], io: Dict[str, Any], workers: int, label: str, +) -> None: + if ( + normalized["workers_used"] != workers + or normalized["container_open_count"] != normalized["container_close_count"] + ): + raise RuntimeError(f"{label} counters inconsistent") + if ( + io["container_opens"] != normalized["container_open_count"] + or io["container_appends"] != normalized["container_append_count"] + or io["fsyncs"] != normalized["fsync_count"] + ): + raise RuntimeError(f"{label} duplicated counters inconsistent") + if normalized["total_files"] <= 0 or normalized["total_bytes"] <= 0: + raise RuntimeError(f"{label} logical totals invalid") + + +def validate_execution_stat_io_contract( + normalized: Dict[str, Any], io: Dict[str, Any], label: str, +) -> None: + if normalized["container_append_count"] > 0 and ( + normalized["container_open_count"] == 0 + or normalized["fsync_count"] == 0 + or io["bytes_written"] == 0 + ): + raise RuntimeError(f"{label} append counters contradict I/O") + if io["bytes_read"] > 0 and normalized["container_open_count"] == 0: + raise RuntimeError(f"{label} read counters contradict container opens") + + +def validate_execution_stat_snapshot( + normalized: Dict[str, Any], case_name: Optional[str], label: str, +) -> None: + if ( + case_name is not None + and normalized["snapshot_metadata_write_count"] > 0 + and case_name not in {"snapshot-creation", "gc-after-churn"} + ): + raise RuntimeError(f"{label} snapshot writes contradict operation") + + +def validate_timing_envelope(envelope: Dict[str, Any], *, workers: int, legacy: bool) -> None: + require_fields(envelope, {"status", "command", "data"}, "benchmark envelope") + if envelope["status"] != "ok" or envelope["command"] != "benchmark": + raise RuntimeError("benchmark envelope is not successful") + data = validate_timing_data(envelope["data"], workers, legacy) + if not legacy: + validate_timing_fixture(data["fixture"]) + row_stats = validate_timing_rows(data["rows"], workers, legacy) + validate_timing_totals(data["execution_stats"], row_stats, workers) + + +def validate_timing_data(value: Any, workers: int, legacy: bool) -> Dict[str, Any]: + data_fields = {"generated_at_utc", "dataset", "repeat", "execution", "execution_stats", "rows"} + if not legacy: + data_fields |= {"schema_version", "fixture"} + data = require_fields(value, data_fields, "benchmark data") + if data.get("dataset") != "small" or data.get("repeat") != 1: + raise RuntimeError("benchmark dataset/repeat mismatch") + if not legacy and data.get("schema_version") != 2: + raise RuntimeError("candidate benchmark schema must be 2") + validate_execution(data["execution"], workers, "benchmark execution") + return data + + +def validate_timing_fixture(value: Any) -> None: + fixture = require_fields(value, set(SMALL_FIXTURE) | {"ordered_cases"}, "fixture") + validate_timing_fixture_fields(fixture) + validate_timing_fixture_order(fixture["ordered_cases"]) + + +def validate_timing_fixture_fields(fixture: Dict[str, Any]) -> None: + for field, expected in SMALL_FIXTURE.items(): + if fixture[field] != expected: + raise RuntimeError(f"fixture field {field} mismatch") + + +def validate_timing_fixture_order(ordered: Any) -> None: + if not isinstance(ordered, list) or [item.get("name") for item in ordered] != EXPECTED_CASES: + raise RuntimeError("fixture case order mismatch") + if [item.get("seed") for item in ordered] != [1712 + index * 10 for index in range(9)]: + raise RuntimeError("fixture case seed mismatch") + + +def validate_timing_rows(rows: Any, workers: int, legacy: bool) -> List[Dict[str, Any]]: + if ( + not isinstance(rows, list) + or any(not isinstance(row, dict) for row in rows) + or [row.get("case") for row in rows] != EXPECTED_CASES + ): + raise RuntimeError("benchmark row case order mismatch") + row_fields = TIMING_ROW_REQUIRED_FIELDS + row_stats: List[Dict[str, Any]] = [] + for row in rows: + row_stats.append(validate_timing_row(row, row_fields, workers, legacy)) + return row_stats + + +def validate_timing_row( + row: Any, row_fields: set[str], workers: int, legacy: bool, +) -> Dict[str, Any]: + row = require_timing_row(row, row_fields, legacy) + validate_execution(row["execution"], workers, f"row {row['case']} execution") + duration = row["duration_ms"] + throughput = row["throughput_mbps"] + if not positive_finite_number(duration) or not positive_finite_number(throughput): + raise RuntimeError(f"row {row['case']} timing values invalid") + stats = validate_execution_stats( + row["execution_stats"], + workers=workers, + label=f"row {row['case']} execution_stats", + case_name=row["case"], + ) + validate_timing_row_diagnostic(row, legacy) + validate_timing_row_throughput(row, stats, legacy) + return stats + + +def require_timing_row(row: Any, row_fields: set[str], legacy: bool) -> Dict[str, Any]: + if ( + not isinstance(row, dict) + or not row_fields <= set(row) + or set(row) - row_fields - (set() if legacy else TIMING_ROW_OPTIONAL_FIELDS) + ): + case_name = row.get("case") if isinstance(row, dict) else None + raise RuntimeError(f"row {case_name} fields mismatch") + return row + + +def validate_timing_row_diagnostic(row: Dict[str, Any], legacy: bool) -> None: + if not legacy and "diagnostic_final_state" in row: + benchmark_contract.validate_diagnostic_final_state( + row["diagnostic_final_state"], f"row {row['case']}" + ) + + +def validate_timing_row_throughput( + row: Dict[str, Any], stats: Dict[str, Any], legacy: bool, +) -> None: + expected_throughput = ( + stats["total_bytes"] / (1024 * 1024) / (float(row["duration_ms"]) / 1000.0) + ) + if not legacy and not math.isclose( + float(row["throughput_mbps"]), expected_throughput, rel_tol=1e-12, abs_tol=1e-12 + ): + raise RuntimeError(f"row {row['case']} throughput is not derived from bytes and duration") + + +def positive_finite_number(value: Any) -> bool: + return ( + not isinstance(value, bool) + and isinstance(value, (int, float)) + and math.isfinite(float(value)) + and float(value) > 0 + ) + + +def validate_timing_totals( + value: Any, row_stats: List[Dict[str, Any]], workers: int, +) -> None: + totals = validate_execution_stats( + value, workers=workers, label="benchmark execution_stats" + ) + sum_fields = { + "total_files", "total_bytes", "container_append_count", "fsync_count", + "container_open_count", "container_close_count", "snapshot_metadata_write_count", + } + for field in sum_fields: + if totals.get(field, 0) != sum(stats.get(field, 0) for stats in row_stats): + raise RuntimeError(f"benchmark execution_stats {field} total mismatch") + for field in totals["io"]: + if totals["io"][field] != sum(stats["io"][field] for stats in row_stats): + raise RuntimeError(f"benchmark execution_stats I/O {field} total mismatch") + + def get_threshold( thresholds_config: Dict[str, Any], mode: str, @@ -291,7 +585,228 @@ def is_hard_fail_check( return default_stage != '1_warnings_only' -def main(): +def advisory_report( + *, + mode: str, + result_path: pathlib.Path, + baseline_path: pathlib.Path, + violations: List[Dict[str, Any]], +) -> Dict[str, Any]: + classification = ( + "BENCHMARK_TIMING_WARNING" if violations else "BENCHMARK_TIMING_WITHIN_REFERENCE" + ) + return { + "schema_version": 1, + "report_kind": ADVISORY_REPORT_KIND, + "status": "complete", + "classification": classification, + "authority": "informational", + "reference_kind": "historical_v1.9_absolute", + "mode": mode, + "candidate_sha256": sha256_file(result_path), + "baseline_sha256": sha256_file(baseline_path), + "violations_count": len(violations), + "violations": violations, + } + + +def advisory_failure_report( + arguments: List[str], error: Exception, +) -> Optional[Tuple[pathlib.Path, Dict[str, Any]]]: + if not arguments or arguments[0] != "check": + return None + report_option = advisory_argument_option(arguments, "--json-report") + if ( + advisory_argument_option(arguments, "--policy") != "hosted-advisory" + or report_option is None + ): + return None + report_path = pathlib.Path(report_option) + category, message = advisory_failure_classification(error) + report = advisory_failure_body(category, message) + mode = advisory_argument_option(arguments, "--mode") + if mode in {"uncompressed", "compressed"}: + report["mode"] = mode + add_advisory_failure_hashes(report, arguments) + return report_path, report + + +def advisory_argument_option(arguments: List[str], name: str) -> Optional[str]: + try: + index = arguments.index(name) + except ValueError: + return None + return arguments[index + 1] if index + 1 < len(arguments) else None + + +def advisory_failure_classification(error: Exception) -> Tuple[str, str]: + if isinstance(error, FileNotFoundError): + return "missing_input", "required benchmark advisory input is missing" + if isinstance(error, (json.JSONDecodeError, yaml.YAMLError)): + return "malformed_input", "benchmark advisory input is malformed" + if isinstance(error, RuntimeError) and str(error).startswith("No valid benchmark envelope"): + return "malformed_input", "benchmark advisory input is malformed" + if isinstance(error, OSError): + return "io_error", "benchmark advisory input or report I/O failed" + if isinstance(error, ValueError): + return "configuration_error", "benchmark advisory configuration is invalid" + return "contract_error", "benchmark advisory evidence violates its contract" + + +def advisory_failure_body(category: str, message: str) -> Dict[str, Any]: + return { + "schema_version": 1, + "report_kind": ADVISORY_REPORT_KIND, + "status": "failed", + "classification": "BENCHMARK_TIMING_EVALUATION_FAILURE", + "authority": "informational", + "reference_kind": "historical_v1.9_absolute", + "error": {"category": category, "message": message}, + "violations_count": 0, + "violations": [], + } + + +def add_advisory_failure_hashes(report: Dict[str, Any], arguments: List[str]) -> None: + baseline_option = advisory_argument_option(arguments, "--baseline") + paths = { + "candidate_sha256": pathlib.Path(arguments[1]) if len(arguments) > 1 else None, + "baseline_sha256": pathlib.Path(baseline_option) if baseline_option else None, + } + for field, path in paths.items(): + if path is not None: + try: + report[field] = sha256_file(path) + except OSError: + pass + + +def validate_advisory_report(report: Any) -> Dict[str, Any]: + report, classification = require_advisory_report_fields(report) + validate_advisory_identity(report, classification) + validate_advisory_status(report, classification) + validate_advisory_hashes(report) + validate_advisory_violation_inventory(report) + validate_advisory_classification_inventory(report, classification) + validate_advisory_failure_error(report, classification) + return report + + +def require_advisory_report_fields(report: Any) -> Tuple[Dict[str, Any], Any]: + if not isinstance(report, dict): + raise RuntimeError("timing advisory report fields mismatch") + classification = report.get("classification") + common_fields = { + "schema_version", "report_kind", "status", "classification", "authority", + "reference_kind", "violations_count", "violations", + } + if classification == "BENCHMARK_TIMING_EVALUATION_FAILURE": + required = common_fields | {"error"} + optional = {"mode", "candidate_sha256", "baseline_sha256"} + if not required <= set(report) or set(report) - required - optional: + raise RuntimeError("timing advisory report fields mismatch") + else: + report = require_fields( + report, + common_fields | {"mode", "candidate_sha256", "baseline_sha256"}, + "timing advisory report", + ) + return report, classification + + +def validate_advisory_identity(report: Dict[str, Any], classification: Any) -> None: + if ( + report["schema_version"] != 1 + or report["report_kind"] != ADVISORY_REPORT_KIND + or report["status"] not in {"complete", "not_evaluated", "failed"} + ): + raise RuntimeError("timing advisory report identity mismatch") + validate_advisory_authority_identity(report, classification) + + +def validate_advisory_authority_identity( + report: Dict[str, Any], classification: Any, +) -> None: + if ( + classification not in ADVISORY_EXIT_CODES + or report["authority"] != "informational" + or report["reference_kind"] != "historical_v1.9_absolute" + ): + raise RuntimeError("timing advisory report identity mismatch") + if "mode" in report and report["mode"] not in {"uncompressed", "compressed"}: + raise RuntimeError("timing advisory report identity mismatch") + + +def validate_advisory_status(report: Dict[str, Any], classification: Any) -> None: + expected_status = { + "BENCHMARK_TIMING_NOT_EVALUATED": "not_evaluated", + "BENCHMARK_TIMING_EVALUATION_FAILURE": "failed", + }.get(classification, "complete") + if report["status"] != expected_status: + raise RuntimeError("timing advisory report status mismatch") + + +def validate_advisory_hashes(report: Dict[str, Any]) -> None: + for field in ("candidate_sha256", "baseline_sha256"): + if field not in report: + continue + value = report[field] + if ( + not isinstance(value, str) + or len(value) != 64 + or any(ch not in "0123456789abcdef" for ch in value) + ): + raise RuntimeError(f"timing advisory {field} invalid") + + +def validate_advisory_violation_inventory(report: Dict[str, Any]) -> None: + if ( + isinstance(report["violations_count"], bool) + or not isinstance(report["violations_count"], int) + or report["violations_count"] < 0 + or not isinstance(report["violations"], list) + or report["violations_count"] != len(report["violations"]) + ): + raise RuntimeError("timing advisory violation inventory mismatch") + + +def validate_advisory_classification_inventory( + report: Dict[str, Any], classification: Any, +) -> None: + if classification == "BENCHMARK_TIMING_WITHIN_REFERENCE" and report["violations"]: + raise RuntimeError("within-reference advisory contains violations") + if classification == "BENCHMARK_TIMING_WARNING" and not report["violations"]: + raise RuntimeError("timing warning contains no violations") + if ( + classification + in {"BENCHMARK_TIMING_NOT_EVALUATED", "BENCHMARK_TIMING_EVALUATION_FAILURE"} + and report["violations"] + ): + raise RuntimeError("unevaluated timing report contains violations") + + +def validate_advisory_failure_error(report: Dict[str, Any], classification: Any) -> None: + if classification == "BENCHMARK_TIMING_EVALUATION_FAILURE": + error = require_fields(report["error"], {"category", "message"}, "timing advisory error") + if ( + error["category"] not in ADVISORY_FAILURE_CATEGORIES + or not isinstance(error["message"], str) + or not error["message"] + ): + raise RuntimeError("timing advisory error invalid") + + +def verify_advisory_exit(report_path: pathlib.Path, observed_exit_code: int) -> int: + report = validate_advisory_report(json.loads(report_path.read_text(encoding="utf-8"))) + expected = ADVISORY_EXIT_CODES[report["classification"]] + if observed_exit_code != expected: + raise RuntimeError( + f"timing advisory exit mismatch: classification expects {expected}, got {observed_exit_code}" + ) + return 0 + + +def _main(): parser = argparse.ArgumentParser( description="Validate benchmark regression thresholds" ) @@ -349,6 +864,19 @@ def main(): type=pathlib.Path, help='Write JSON report to file' ) + check_parser.add_argument( + '--policy', + choices=['legacy', 'hosted-advisory'], + default='legacy', + help='Select legacy failure behavior or hosted informational timing policy' + ) + + verify_parser = subparsers.add_parser( + 'verify-advisory-exit', + help='Verify an advisory report classification against the observed comparator exit' + ) + verify_parser.add_argument('--report', type=pathlib.Path, required=True) + verify_parser.add_argument('--observed-exit-code', type=int, required=True) args = parser.parse_args() @@ -373,6 +901,15 @@ def main(): thresholds_config = load_yaml(args.thresholds) result_envelope = load_benchmark_envelope(args.result) baseline_envelope = load_benchmark_envelope(args.baseline) + advisory = args.policy == 'hosted-advisory' + if advisory: + result_execution = result_envelope.get('data', {}).get('execution', {}) + baseline_execution = baseline_envelope.get('data', {}).get('execution', {}) + workers = result_execution.get('store_folder_workers') + if workers not in {1, 4} or baseline_execution.get('store_folder_workers') != workers: + raise RuntimeError('candidate/baseline worker profile mismatch') + validate_timing_envelope(result_envelope, workers=workers, legacy=False) + validate_timing_envelope(baseline_envelope, workers=workers, legacy=True) passed, violations, has_hard_fails = check_regression( result_envelope, baseline_envelope, thresholds_config, args.mode @@ -382,7 +919,8 @@ def main(): if not violations: print(f"✓ No regressions detected (mode: {args.mode})") else: - print(f"\n{'HARD FAIL' if has_hard_fails else 'WARNINGS'}: " + label = 'TIMING WARNING' if advisory else ('HARD FAIL' if has_hard_fails else 'WARNINGS') + print(f"\n{label}: " f"{len(violations)} regression(s) detected (mode: {args.mode})\n") # Group by severity @@ -404,29 +942,65 @@ def print_violations(label, vlist): print(f" {v['case']:<30} {v['metric']:<12} {base_str:<12} " f"{result_str:<12} {delta_str:<10} {threshold_str:<10}") - if hard_fails: - print_violations("HARD FAIL", hard_fails) - if warnings: - print_violations("WARNING", warnings) + if advisory: + print_violations("TIMING ADVISORY", violations) + else: + if hard_fails: + print_violations("HARD FAIL", hard_fails) + if warnings: + print_violations("WARNING", warnings) # Write JSON report if requested if args.json_report: - report = { + report = advisory_report( + mode=args.mode, + result_path=args.result, + baseline_path=args.baseline, + violations=violations, + ) if advisory else { 'mode': args.mode, 'passed': passed, 'has_hard_fails': has_hard_fails, 'violations_count': len(violations), 'violations': violations, } + if advisory: + validate_advisory_report(report) args.json_report.write_text(json.dumps(report, indent=2) + '\n') print(f"\nReport written to {args.json_report}") - + + if advisory: + if not args.json_report: + raise RuntimeError('hosted-advisory policy requires --json-report') + return ADVISORY_EXIT_CODES[report['classification']] return 0 if passed else 1 + + elif args.command == 'verify-advisory-exit': + return verify_advisory_exit(args.report, args.observed_exit_code) else: parser.print_help() return 2 +def main() -> int: + try: + return _main() + except ( + AttributeError, KeyError, OSError, TypeError, ValueError, RuntimeError, + json.JSONDecodeError, yaml.YAMLError, + ) as exc: + failure = advisory_failure_report(sys.argv[1:], exc) + if failure is not None: + report_path, report = failure + try: + validate_advisory_report(report) + report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError, RuntimeError) as report_exc: + print(f"benchmark timing failure-report error: {report_exc}", file=sys.stderr) + print(f"benchmark timing policy error: {exc}", file=sys.stderr) + return 2 + + if __name__ == '__main__': sys.exit(main()) diff --git a/tests/adversarial/g6_concurrent_operations_adversarial_test.go b/tests/adversarial/g6_concurrent_operations_adversarial_test.go index 114c4cb4..ee027e2f 100644 --- a/tests/adversarial/g6_concurrent_operations_adversarial_test.go +++ b/tests/adversarial/g6_concurrent_operations_adversarial_test.go @@ -1,11 +1,15 @@ package main import ( + "bufio" + "bytes" "context" "crypto/sha256" "database/sql" "encoding/hex" + "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -18,7 +22,9 @@ import ( dbschema "github.com/franchoy/coldkeep/db" "github.com/franchoy/coldkeep/internal/blocks" + "github.com/franchoy/coldkeep/internal/chunk" "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/coordination" "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/maintenance" "github.com/franchoy/coldkeep/internal/storage" @@ -30,16 +36,17 @@ import ( // G6 — Safe concurrent storage operations // // Adversarial goals: -// - concurrent store/remove/GC activity must not corrupt metadata or graph shape -// - identical concurrent stores must converge on deterministic chunk graphs +// - repository coordination must reject known independent-process overlap +// - repeated same-file CLI stores must converge on deterministic chunk graphs +// - lower-layer concurrent store/remove/GC activity must not corrupt metadata or graph shape // - mixed concurrent operations must preserve healthy restores for surviving files // - verification invariants must remain true after each stress phase // // Notes: // - This file uses the current Postgres-backed adversarial harness. // - It runs a codec matrix for plain + aes-gcm where data-path behavior matters. -// - It intentionally validates semantics after concurrency rather than imposing -// scheduler-specific timing assumptions. +// - Independent-process contention uses an explicit holder READY protocol; +// storage-level interleaving tests retain the internal concurrency evidence. func adversarialG6Codecs() []string { return []string{"plain", "aes-gcm"} @@ -520,29 +527,499 @@ func storeFileWithCodecCLIG6(t *testing.T, repoRoot, binPath string, env map[str return testutils.JSONInt64(t, data, "file_id") } +const ( + g6LeaseHolderEnv = "COLDKEEP_G6_REPOSITORY_LEASE_HOLDER" + g6LeaseHolderReady = "READY" + g6LeaseHolderRelease = "RELEASE" + g6LeaseHolderDone = "RELEASED" +) + +// TestAdversarialG6RepositoryLeaseHolderProcess is a helper-process entry +// point. The parent test executes this test binary in a separate OS process +// with g6LeaseHolderEnv set, so acquisition uses the production Coordinator +// while synchronization remains test-only. +func TestAdversarialG6RepositoryLeaseHolderProcess(t *testing.T) { + if strings.TrimSpace(os.Getenv(g6LeaseHolderEnv)) != "1" { + return + } + + identity, err := coordination.ResolveIdentity(os.Getenv("COLDKEEP_STORAGE_DIR")) + if err != nil { + t.Fatalf("resolve holder repository identity: %v", err) + } + owner, err := coordination.NewOwner(coordination.OperationStore, identity, "phase13a-test", time.Now()) + if err != nil { + t.Fatalf("create holder owner metadata: %v", err) + } + lease, err := coordination.NewCoordinator().Acquire(context.Background(), identity, coordination.Request{ + Operation: coordination.OperationStore, + Mode: coordination.ModeExclusive, + Owner: owner, + }) + if err != nil { + t.Fatalf("acquire holder repository Lease: %v", err) + } + released := false + defer func() { + if !released { + _ = lease.Release() + } + }() + + if _, err := fmt.Fprintln(os.Stdout, g6LeaseHolderReady); err != nil { + t.Fatalf("signal holder readiness: %v", err) + } + scanner := bufio.NewScanner(os.Stdin) + if !scanner.Scan() { + if err := scanner.Err(); err != nil { + t.Fatalf("read holder release signal: %v", err) + } + t.Fatal("holder release signal stream closed before RELEASE") + } + if signal := strings.TrimSpace(scanner.Text()); signal != g6LeaseHolderRelease { + t.Fatalf("holder release signal=%q want %q", signal, g6LeaseHolderRelease) + } + if err := lease.Release(); err != nil { + t.Fatalf("release holder repository Lease: %v", err) + } + released = true + if _, err := fmt.Fprintln(os.Stdout, g6LeaseHolderDone); err != nil { + t.Fatalf("signal holder release: %v", err) + } +} + +type g6RepositoryLeaseHolder struct { + command *exec.Cmd + stdin io.WriteCloser + lines chan string + wait chan error + stderr *bytes.Buffer + finished bool +} + +func startG6RepositoryLeaseHolder(t *testing.T, env map[string]string) *g6RepositoryLeaseHolder { + t.Helper() + + testBinary, err := os.Executable() + if err != nil { + t.Fatalf("resolve G6 test binary: %v", err) + } + holderEnv := make(map[string]string, len(env)+1) + for key, value := range env { + holderEnv[key] = value + } + // The holder does not open the database. Removing this gate prevents the + // adversarial package TestMain from creating a second isolated PostgreSQL + // database inside the helper process. + holderEnv["COLDKEEP_TEST_DB"] = "" + holderEnv[g6LeaseHolderEnv] = "1" + + command := exec.Command(testBinary, "-test.run=^TestAdversarialG6RepositoryLeaseHolderProcess$", "-test.count=1") + command.Env = testutils.BuildCommandEnv(holderEnv) + stdin, err := command.StdinPipe() + if err != nil { + t.Fatalf("create holder stdin pipe: %v", err) + } + stdout, err := command.StdoutPipe() + if err != nil { + _ = stdin.Close() + t.Fatalf("create holder stdout pipe: %v", err) + } + stderr := &bytes.Buffer{} + command.Stderr = stderr + if err := command.Start(); err != nil { + _ = stdin.Close() + t.Fatalf("start repository Lease holder: %v", err) + } + + holder := &g6RepositoryLeaseHolder{ + command: command, + stdin: stdin, + lines: make(chan string, 16), + wait: make(chan error, 1), + stderr: stderr, + } + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + holder.lines <- strings.TrimSpace(scanner.Text()) + } + close(holder.lines) + }() + go func() { + holder.wait <- command.Wait() + }() + t.Cleanup(func() { holder.cleanup(t) }) + holder.waitForLine(t, g6LeaseHolderReady) + return holder +} + +func (holder *g6RepositoryLeaseHolder) waitForLine(t *testing.T, want string) { + t.Helper() + deadline := time.NewTimer(10 * time.Second) + defer deadline.Stop() + for { + select { + case line, ok := <-holder.lines: + if !ok { + select { + case err := <-holder.wait: + holder.finished = true + t.Fatalf("repository Lease holder exited before %q: %v; stderr=%s", want, err, holder.stderr.String()) + case <-time.After(2 * time.Second): + t.Fatalf("repository Lease holder output closed before %q", want) + } + } + if line == want { + return + } + case <-deadline.C: + t.Fatalf("timeout waiting for repository Lease holder signal %q", want) + } + } +} + +func (holder *g6RepositoryLeaseHolder) release(t *testing.T) { + t.Helper() + if holder.finished { + t.Fatal("repository Lease holder exited before release") + } + if _, err := fmt.Fprintln(holder.stdin, g6LeaseHolderRelease); err != nil { + t.Fatalf("send repository Lease holder release: %v", err) + } + if err := holder.stdin.Close(); err != nil { + t.Fatalf("close repository Lease holder stdin: %v", err) + } + holder.waitForLine(t, g6LeaseHolderDone) + select { + case err := <-holder.wait: + holder.finished = true + if err != nil { + t.Fatalf("repository Lease holder exit: %v; stderr=%s", err, holder.stderr.String()) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for repository Lease holder exit") + } +} + +func (holder *g6RepositoryLeaseHolder) kill(t *testing.T) { + t.Helper() + if holder.finished { + t.Fatal("repository Lease holder exited before intentional kill") + } + if holder.command.Process == nil { + t.Fatal("repository Lease holder has no process to kill") + } + if err := holder.command.Process.Kill(); err != nil { + t.Fatalf("kill repository Lease holder: %v", err) + } + _ = holder.stdin.Close() + + select { + case err := <-holder.wait: + holder.finished = true + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("repository Lease holder wait error=%T %v want killed-process ExitError; stderr=%s", err, err, holder.stderr.String()) + } + if exitErr.ProcessState == nil || exitErr.Success() { + t.Fatalf("repository Lease holder process state=%v want unsuccessful killed exit; stderr=%s", exitErr.ProcessState, holder.stderr.String()) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for killed repository Lease holder exit") + } +} + +func (holder *g6RepositoryLeaseHolder) cleanup(t *testing.T) { + t.Helper() + if holder == nil || holder.finished { + return + } + _, _ = fmt.Fprintln(holder.stdin, g6LeaseHolderRelease) + _ = holder.stdin.Close() + select { + case <-holder.wait: + holder.finished = true + return + case <-time.After(2 * time.Second): + } + if holder.command.Process != nil { + _ = holder.command.Process.Kill() + } + select { + case <-holder.wait: + holder.finished = true + case <-time.After(5 * time.Second): + t.Log("repository Lease holder did not exit after cleanup kill") + } +} + +func runColdkeepCommandWithTimeoutG6( + t *testing.T, + repoRoot string, + binPath string, + env map[string]string, + args ...string, +) testutils.CLIExecResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + command := exec.CommandContext(ctx, binPath, args...) + command.Dir = repoRoot + command.Env = testutils.BuildCommandEnv(env) + var stdout bytes.Buffer + var stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + err := command.Run() + if ctx.Err() != nil { + t.Fatalf("coldkeep command %v timed out: %v", args, ctx.Err()) + } + if err == nil { + return testutils.CLIExecResult{Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: 0} + } + if exitErr, ok := err.(*exec.ExitError); ok { + return testutils.CLIExecResult{Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: exitErr.ExitCode()} + } + t.Fatalf("run coldkeep command %v: %v", args, err) + return testutils.CLIExecResult{} +} + +type g6AsyncCLICommand struct { + command *exec.Cmd + wait chan error + stdout *bytes.Buffer + stderr *bytes.Buffer + finished bool +} + +func startG6AsyncCLICommand( + t *testing.T, + repoRoot string, + binPath string, + env map[string]string, + args ...string, +) *g6AsyncCLICommand { + t.Helper() + + command := exec.Command(binPath, args...) + command.Dir = repoRoot + command.Env = testutils.BuildCommandEnv(env) + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + command.Stdout = stdout + command.Stderr = stderr + if err := command.Start(); err != nil { + t.Fatalf("start coldkeep command %v: %v", args, err) + } + + running := &g6AsyncCLICommand{ + command: command, + wait: make(chan error, 1), + stdout: stdout, + stderr: stderr, + } + go func() { running.wait <- command.Wait() }() + t.Cleanup(func() { running.cleanup(t) }) + return running +} + +func (running *g6AsyncCLICommand) result(t *testing.T) testutils.CLIExecResult { + t.Helper() + if running.finished { + t.Fatal("coldkeep command result requested after process was already reaped") + } + select { + case err := <-running.wait: + running.finished = true + if err == nil { + return testutils.CLIExecResult{Stdout: running.stdout.String(), Stderr: running.stderr.String(), ExitCode: 0} + } + if exitErr, ok := err.(*exec.ExitError); ok { + return testutils.CLIExecResult{ + Stdout: running.stdout.String(), + Stderr: running.stderr.String(), + ExitCode: exitErr.ExitCode(), + } + } + t.Fatalf("wait for coldkeep command: %v; stdout=%s stderr=%s", err, running.stdout.String(), running.stderr.String()) + case <-time.After(30 * time.Second): + if running.command.Process != nil { + _ = running.command.Process.Kill() + } + t.Fatalf("timeout waiting for coldkeep command; stdout=%s stderr=%s", running.stdout.String(), running.stderr.String()) + } + return testutils.CLIExecResult{} +} + +func (running *g6AsyncCLICommand) cleanup(t *testing.T) { + t.Helper() + if running == nil || running.finished { + return + } + if running.command.Process != nil { + _ = running.command.Process.Kill() + } + select { + case <-running.wait: + running.finished = true + case <-time.After(5 * time.Second): + t.Log("coldkeep subprocess did not exit after cleanup kill") + } +} + +func waitForG6GCPhysicalGraphLockWait(t *testing.T, observer *sql.Conn, blockerPID int) int { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for { + var waiterPID int + err := observer.QueryRowContext(ctx, ` + SELECT activity.pid + FROM pg_stat_activity activity + WHERE activity.datname = current_database() + AND activity.state = 'active' + AND activity.wait_event_type = 'Lock' + AND $1 = ANY(pg_blocking_pids(activity.pid)) + AND activity.query LIKE '%FROM physical_file pf%' + ORDER BY activity.pid + LIMIT 1 + `, blockerPID).Scan(&waiterPID) + if err == nil { + return waiterPID + } + if !errors.Is(err, sql.ErrNoRows) && ctx.Err() == nil { + t.Fatalf("observe live-GC physical-graph lock wait: %v", err) + } + select { + case <-ctx.Done(): + t.Fatalf("live GC did not enter the server-observed physical-graph lock wait: %v", ctx.Err()) + case <-ticker.C: + } + } +} + +func assertRepositoryBusyCLIResultG6(t *testing.T, result testutils.CLIExecResult) { + t.Helper() + if result.ExitCode != 1 { + t.Fatalf("busy contender exit=%d want 1; stdout=%s stderr=%s", result.ExitCode, result.Stdout, result.Stderr) + } + if strings.TrimSpace(result.Stdout) != "" { + t.Fatalf("busy contender stdout=%q want empty", result.Stdout) + } + payload, ok := testutils.TryParseLastJSONLine(result.Stderr) + if !ok { + t.Fatalf("busy contender produced no JSON error; stderr=%s", result.Stderr) + } + if got, _ := payload["status"].(string); got != "error" { + t.Fatalf("busy status=%q want error; payload=%v", got, payload) + } + if got, _ := payload["error_class"].(string); got != "GENERAL" { + t.Fatalf("busy error_class=%q want GENERAL; payload=%v", got, payload) + } + if got, _ := payload["exit_code"].(float64); int(got) != 1 { + t.Fatalf("busy JSON exit_code=%v want 1; payload=%v", payload["exit_code"], payload) + } + if got, _ := payload["message"].(string); got != "repository is busy" { + t.Fatalf("busy message=%q want %q; payload=%v", got, "repository is busy", payload) + } + errorNode, ok := payload["error"].(map[string]any) + if !ok { + t.Fatalf("busy error node=%T want object; payload=%v", payload["error"], payload) + } + if got, _ := errorNode["code"].(string); got != "REPOSITORY_BUSY" { + t.Fatalf("busy error code=%q want REPOSITORY_BUSY; payload=%v", got, payload) + } + if got, _ := errorNode["message"].(string); got != "repository is busy" { + t.Fatalf("busy nested message=%q want %q; payload=%v", got, "repository is busy", payload) + } +} + // storeFileWithCodecCLIG6Async is safe to call from goroutines because it // returns an error instead of calling t.Fatal/t.FailNow. func storeFileWithCodecCLIG6Async(repoRoot, binPath string, env map[string]string, codec, path string) (int64, error) { + result, err := storeFileWithCodecCLIG6AsyncDiagnostics(repoRoot, binPath, env, codec, path) + return result.FileID, err +} + +type g6CLIStoreCommandResult struct { + FileID int64 + LifecycleTrace []string + StartedUTC time.Time + FinishedUTC time.Time +} + +func storeFileWithCodecCLIG6AsyncDiagnostics(repoRoot, binPath string, env map[string]string, codec, path string) (g6CLIStoreCommandResult, error) { + result := g6CLIStoreCommandResult{StartedUTC: time.Now().UTC()} cmd := exec.Command(binPath, "store", "--codec", codec, path, "--output", "json") cmd.Dir = repoRoot cmd.Env = testutils.BuildCommandEnv(env) out, err := cmd.CombinedOutput() + result.FinishedUTC = time.Now().UTC() + result.LifecycleTrace = filterG6LifecycleTrace(string(out), env) if err != nil { - return 0, fmt.Errorf("store command: %w; output=%s", err, out) + return result, fmt.Errorf("store command: %w; output=%s", err, sanitizeG6DiagnosticText(string(out), env)) } payload, ok := testutils.TryParseLastJSONLine(string(out)) if !ok { - return 0, fmt.Errorf("no JSON in store output: %s", out) + return result, fmt.Errorf("no JSON in store output: %s", sanitizeG6DiagnosticText(string(out), env)) } data, ok := payload["data"].(map[string]any) if !ok { - return 0, fmt.Errorf("store payload missing data: %v", payload) + return result, fmt.Errorf("store payload missing data: %v", payload) } idF, ok := data["file_id"].(float64) if !ok { - return 0, fmt.Errorf("store payload missing file_id: %v", data) + return result, fmt.Errorf("store payload missing file_id: %v", data) + } + result.FileID = int64(idF) + return result, nil +} + +var g6LifecycleEventMarkers = []string{ + "event=store_reuse_claim_graph_invalid", + "event=store_reuse_validation_failed", + "event=chunk_reuse_validation_failed", + "event=store_chunk_reclaim", +} + +func filterG6LifecycleTrace(output string, env map[string]string) []string { + const maxTraceLines = 256 + trace := make([]string, 0) + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(raw) + if line == "" || !containsG6LifecycleMarker(line) { + continue + } + trace = append(trace, sanitizeG6DiagnosticText(line, env)) + if len(trace) == maxTraceLines { + break + } } - return int64(idF), nil + return trace +} + +func containsG6LifecycleMarker(line string) bool { + for _, marker := range g6LifecycleEventMarkers { + if strings.Contains(line, marker) { + return true + } + } + return false +} + +func sanitizeG6DiagnosticText(value string, env map[string]string) string { + for _, key := range []string{"COLDKEEP_KEY", "DB_PASSWORD"} { + secret := strings.TrimSpace(env[key]) + if secret != "" { + value = strings.ReplaceAll(value, secret, "[REDACTED]") + } + } + return value } func restoreMustMatchHashG6(t *testing.T, dbconn *sql.DB, fileID int64, outPath, wantHash string) { @@ -587,7 +1064,7 @@ func verifyConcurrentInvariantsG6(t *testing.T, dbconn *sql.DB, diag *g6FailureD } } -var g6ChunkIDPattern = regexp.MustCompile(`chunk (\d+)`) +var g6ChunkIDPattern = regexp.MustCompile(`chunk(?:[ =])(\d+)`) type g6FailureDiagnosticContext struct { TestName string @@ -602,9 +1079,12 @@ type g6FailureDiagnosticContext struct { } type g6StoreOperationResult struct { - Worker int `json:"worker"` - FileID int64 `json:"file_id,omitempty"` - Error string `json:"error,omitempty"` + Worker int `json:"worker"` + FileID int64 `json:"file_id,omitempty"` + Error string `json:"error,omitempty"` + LifecycleTrace []string `json:"lifecycle_trace,omitempty"` + StartedUTC time.Time `json:"started_utc"` + FinishedUTC time.Time `json:"finished_utc"` } type g6ChunkFailureDiagnosticManifest struct { @@ -625,6 +1105,58 @@ type g6ChunkFailureDiagnosticManifest struct { StoreResults []g6StoreOperationResult `json:"store_results"` RelevantConfiguration map[string]string `json:"relevant_configuration,omitempty"` MigrationCompanionState g6ChunkMetadataRecord `json:"migration_companion_state"` + PackedBlocks []g6PackedBlockRecord `json:"packed_blocks,omitempty"` + PhysicalFiles []g6PhysicalFileRecord `json:"physical_files,omitempty"` +} + +type g6PackedBlockRecord struct { + BlockID int64 `json:"block_id"` + FormatVersion int64 `json:"format_version"` + Codec string `json:"codec"` + CompressionCodec string `json:"compression_codec"` + CompressionLevel *int64 `json:"compression_level,omitempty"` + PlaintextSize int64 `json:"plaintext_size"` + CompressedSize *int64 `json:"compressed_size,omitempty"` + StoredSize int64 `json:"stored_size"` + ContainerID int64 `json:"container_id"` + ContainerFilename string `json:"container_filename"` + ContainerMaxSize int64 `json:"container_max_size"` + ContainerOffset int64 `json:"container_offset"` + BlockHash string `json:"block_hash,omitempty"` + PayloadHash string `json:"payload_hash,omitempty"` + CompressedHash string `json:"compressed_hash,omitempty"` + PhysicalHash string `json:"physical_hash,omitempty"` + ActualPhysicalHash string `json:"actual_physical_hash,omitempty"` + ActualPhysicalHashError string `json:"actual_physical_hash_error,omitempty"` + Members []g6PackedBlockMember `json:"members"` + EncodedMembers []g6EncodedBlockMember `json:"encoded_members,omitempty"` + EncodedMembersError string `json:"encoded_members_error,omitempty"` +} + +type g6PackedBlockMember struct { + ChunkID int64 `json:"chunk_id"` + ChunkHash string `json:"chunk_hash"` + ChunkStatus string `json:"chunk_status"` + OffsetInBlock int64 `json:"offset_in_block"` + SizeInBlock int64 `json:"size_in_block"` + LegacyMappingID *int64 `json:"legacy_mapping_id,omitempty"` + LegacyCodec string `json:"legacy_codec,omitempty"` + LegacyContainerID *int64 `json:"legacy_container_id,omitempty"` + LegacyOffset *int64 `json:"legacy_offset,omitempty"` + LegacyStoredSize *int64 `json:"legacy_stored_size,omitempty"` + LegacyNonceLength *int64 `json:"legacy_nonce_length,omitempty"` +} + +type g6EncodedBlockMember struct { + ChunkID uint64 `json:"chunk_id"` + Offset uint64 `json:"offset"` + Size uint64 `json:"size"` +} + +type g6PhysicalFileRecord struct { + ID int64 `json:"id"` + Path string `json:"path"` + LogicalFileID int64 `json:"logical_file_id"` } type g6ChunkMetadataRecord struct { @@ -654,6 +1186,7 @@ func logConcurrentInvariantFailureG6(t *testing.T, dbconn *sql.DB, verifyErr err manifest := buildG6FailureManifest(t, verifyErr, diag) loadG6FailureSchemaVersion(t, dbconn, &manifest) attachG6OffendingChunkMetadata(t, dbconn, verifyErr, &manifest) + attachG6RepositoryState(t, dbconn, &manifest) writeConcurrentInvariantManifestG6(t, manifest) } @@ -935,6 +1468,260 @@ func logContainerMetadataG6(t *testing.T, dbconn *sql.DB, label string, containe return filename } +func attachG6RepositoryState(t *testing.T, dbconn *sql.DB, manifest *g6ChunkFailureDiagnosticManifest) { + t.Helper() + blocks, err := loadG6PackedBlocks(dbconn) + if err != nil { + t.Logf("G6 verify diagnostics: collect packed-block state: %v", err) + } else { + for i := range blocks { + attachG6ActualPhysicalHash(&blocks[i]) + attachG6EncodedBlockMembers(&blocks[i]) + } + manifest.PackedBlocks = blocks + } + + physicalFiles, err := loadG6PhysicalFiles(dbconn) + if err != nil { + t.Logf("G6 verify diagnostics: collect physical-file state: %v", err) + } else { + manifest.PhysicalFiles = physicalFiles + } +} + +func loadG6PackedBlocks(dbconn *sql.DB) ([]g6PackedBlockRecord, error) { + rows, err := dbconn.Query(` + SELECT sb.id, sb.format_version, sb.codec, sb.compression_codec, + sb.compression_level, sb.plaintext_size, sb.compressed_size, + sb.stored_size, sb.container_id, c.filename, c.max_size, sb.container_offset, + sb.block_hash, sb.payload_hash, sb.compressed_hash, sb.physical_hash + FROM storage_blocks sb + JOIN container c ON c.id = sb.container_id + ORDER BY sb.id + `) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + records := make([]g6PackedBlockRecord, 0) + for rows.Next() { + var record g6PackedBlockRecord + var compressionLevel sql.NullInt64 + var compressedSize sql.NullInt64 + var payloadHash sql.NullString + var blockHash []byte + var compressedHash []byte + var physicalHash []byte + if err := rows.Scan( + &record.BlockID, + &record.FormatVersion, + &record.Codec, + &record.CompressionCodec, + &compressionLevel, + &record.PlaintextSize, + &compressedSize, + &record.StoredSize, + &record.ContainerID, + &record.ContainerFilename, + &record.ContainerMaxSize, + &record.ContainerOffset, + &blockHash, + &payloadHash, + &compressedHash, + &physicalHash, + ); err != nil { + return nil, err + } + record.CompressionLevel = nullInt64PointerG6(compressionLevel) + record.CompressedSize = nullInt64PointerG6(compressedSize) + record.BlockHash = hex.EncodeToString(blockHash) + record.PayloadHash = nullStringPlainG6(payloadHash) + record.CompressedHash = hex.EncodeToString(compressedHash) + record.PhysicalHash = hex.EncodeToString(physicalHash) + record.Members, err = loadG6PackedBlockMembers(dbconn, record.BlockID) + if err != nil { + return nil, fmt.Errorf("load members for block %d: %w", record.BlockID, err) + } + records = append(records, record) + } + if err := rows.Err(); err != nil { + return nil, err + } + return records, nil +} + +func loadG6PackedBlockMembers(dbconn *sql.DB, blockID int64) ([]g6PackedBlockMember, error) { + rows, err := dbconn.Query(` + SELECT r.chunk_id, c.chunk_hash, c.status, r.offset_in_block, r.size_in_block, + b.id, b.codec, b.container_id, b.block_offset, b.stored_size, + OCTET_LENGTH(b.nonce) + FROM chunk_block_refs r + JOIN chunk c ON c.id = r.chunk_id + LEFT JOIN blocks b ON b.chunk_id = r.chunk_id + WHERE r.block_id = $1 + ORDER BY r.offset_in_block, r.chunk_id + `, blockID) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + members := make([]g6PackedBlockMember, 0) + for rows.Next() { + var member g6PackedBlockMember + var legacyMappingID sql.NullInt64 + var legacyCodec sql.NullString + var legacyContainerID sql.NullInt64 + var legacyOffset sql.NullInt64 + var legacyStoredSize sql.NullInt64 + var legacyNonceLength sql.NullInt64 + if err := rows.Scan( + &member.ChunkID, + &member.ChunkHash, + &member.ChunkStatus, + &member.OffsetInBlock, + &member.SizeInBlock, + &legacyMappingID, + &legacyCodec, + &legacyContainerID, + &legacyOffset, + &legacyStoredSize, + &legacyNonceLength, + ); err != nil { + return nil, err + } + member.LegacyMappingID = nullInt64PointerG6(legacyMappingID) + member.LegacyCodec = nullStringPlainG6(legacyCodec) + member.LegacyContainerID = nullInt64PointerG6(legacyContainerID) + member.LegacyOffset = nullInt64PointerG6(legacyOffset) + member.LegacyStoredSize = nullInt64PointerG6(legacyStoredSize) + member.LegacyNonceLength = nullInt64PointerG6(legacyNonceLength) + members = append(members, member) + } + if err := rows.Err(); err != nil { + return nil, err + } + return members, nil +} + +func attachG6ActualPhysicalHash(record *g6PackedBlockRecord) { + if record.StoredSize < 0 || record.ContainerOffset < 0 { + record.ActualPhysicalHashError = fmt.Sprintf( + "invalid stored payload bounds: offset=%d size=%d", + record.ContainerOffset, + record.StoredSize, + ) + return + } + if record.ContainerMaxSize > 0 && (record.ContainerOffset > record.ContainerMaxSize || record.StoredSize > record.ContainerMaxSize-record.ContainerOffset) { + record.ActualPhysicalHashError = fmt.Sprintf( + "stored payload exceeds container bounds: offset=%d size=%d max=%d", + record.ContainerOffset, + record.StoredSize, + record.ContainerMaxSize, + ) + return + } + + path, err := container.SafeContainerPath(container.ContainersDir, record.ContainerFilename) + if err != nil { + record.ActualPhysicalHashError = err.Error() + return + } + f, err := os.Open(path) + if err != nil { + record.ActualPhysicalHashError = err.Error() + return + } + defer func() { _ = f.Close() }() + + payload := make([]byte, record.StoredSize) + n, err := f.ReadAt(payload, record.ContainerOffset) + if err != nil { + record.ActualPhysicalHashError = fmt.Sprintf("read stored payload: read=%d expected=%d: %v", n, record.StoredSize, err) + return + } + sum := sha256.Sum256(payload) + record.ActualPhysicalHash = hex.EncodeToString(sum[:]) +} + +func attachG6EncodedBlockMembers(record *g6PackedBlockRecord) { + logicalHash, err := hex.DecodeString(record.BlockHash) + if err != nil { + record.EncodedMembersError = fmt.Sprintf("decode block hash: %v", err) + return + } + compressedHash, err := hex.DecodeString(record.CompressedHash) + if err != nil { + record.EncodedMembersError = fmt.Sprintf("decode compressed hash: %v", err) + return + } + physicalHash, err := hex.DecodeString(record.PhysicalHash) + if err != nil { + record.EncodedMembersError = fmt.Sprintf("decode physical hash: %v", err) + return + } + var compressionLevel *int + if record.CompressionLevel != nil { + value := int(*record.CompressionLevel) + compressionLevel = &value + } + verified, err := verify.VerifyStoredBlock(context.Background(), verify.BlockStorageMetadata{ + BlockID: record.BlockID, + ContainerID: record.ContainerID, + ContainerOffset: record.ContainerOffset, + ContainerName: record.ContainerFilename, + ContainerMaxSize: record.ContainerMaxSize, + FormatVersion: record.FormatVersion, + Codec: record.Codec, + PlaintextSize: record.PlaintextSize, + CompressedSize: record.CompressedSize, + StoredSize: record.StoredSize, + CompressionCodec: record.CompressionCodec, + CompressionLevel: compressionLevel, + LogicalHash: logicalHash, + CompressedHash: compressedHash, + PhysicalHash: physicalHash, + }, verify.FilesystemContainerReader{ContainersDir: container.ContainersDir}) + if err != nil { + record.EncodedMembersError = err.Error() + return + } + if verified == nil || verified.DecodedBlock == nil { + record.EncodedMembersError = "verified block did not include decoded membership" + return + } + for _, entry := range verified.DecodedBlock.Entries { + record.EncodedMembers = append(record.EncodedMembers, g6EncodedBlockMember{ + ChunkID: entry.ChunkID, + Offset: entry.Offset, + Size: entry.Size, + }) + } +} + +func loadG6PhysicalFiles(dbconn *sql.DB) ([]g6PhysicalFileRecord, error) { + rows, err := dbconn.Query(`SELECT id, path, logical_file_id FROM physical_file ORDER BY id`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + records := make([]g6PhysicalFileRecord, 0) + for rows.Next() { + var record g6PhysicalFileRecord + if err := rows.Scan(&record.ID, &record.Path, &record.LogicalFileID); err != nil { + return nil, err + } + records = append(records, record) + } + if err := rows.Err(); err != nil { + return nil, err + } + return records, nil +} + func nullInt64ValueG6(v sql.NullInt64) any { if !v.Valid { return nil @@ -1035,16 +1822,15 @@ func writeConcurrentInvariantManifestG6(t *testing.T, manifest g6ChunkFailureDia t.Logf("G6 verify diagnostics: wrote failure manifest %s", path) } -func TestAdversarialG6ConcurrentStoresSameFileConvergeDeterministically(t *testing.T) { +func TestAdversarialG6IndependentProcessRepositoryContention(t *testing.T) { testgate.RequireDB(t) testgate.RequireLongRun(t) for _, codec := range adversarialG6Codecs() { t.Run(codec, func(t *testing.T) { - outerJobCodec := os.Getenv("COLDKEEP_CODEC") configureAdversarialG6Codec(t, codec) - dbconn, env, repoRoot, binPath, tmp, testDBName := setupAdversarialG6Env(t) + dbconn, env, repoRoot, binPath, tmp, _ := setupAdversarialG6Env(t) defer dbconn.Close() inputDir := filepath.Join(tmp, "input") @@ -1056,50 +1842,290 @@ func TestAdversarialG6ConcurrentStoresSameFileConvergeDeterministically(t *testi t.Fatalf("mkdir restore: %v", err) } - inPath := testutils.CreateTempFile(t, inputDir, "g6-same-file.bin", 2*1024*1024+313) - wantHash := testutils.SHA256File(t, inPath) + inPath := testutils.CreateTempFile(t, inputDir, "g6-contention.bin", 256*1024+313) + fileHash := testutils.SHA256File(t, inPath) + holder := startG6RepositoryLeaseHolder(t, env) + + busyResult := runColdkeepCommandWithTimeoutG6( + t, + repoRoot, + binPath, + env, + "store", "--codec", codec, inPath, "--output", "json", + ) + assertRepositoryBusyCLIResultG6(t, busyResult) - const workers = 6 - type g6storeResult struct { - idx int - id int64 - err error + var storedRows int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM logical_file WHERE file_hash = $1`, fileHash).Scan(&storedRows); err != nil { + t.Fatalf("count logical files after Busy contender: %v", err) } - resultCh := make(chan g6storeResult, workers) - for i := 0; i < workers; i++ { - i := i - go func() { - id, err := storeFileWithCodecCLIG6Async(repoRoot, binPath, env, codec, inPath) - resultCh <- g6storeResult{idx: i, id: id, err: err} - }() + if storedRows != 0 { + t.Fatalf("Busy contender stored %d logical-file rows; want 0", storedRows) } - ids := make([]int64, workers) - storeResults := make([]g6StoreOperationResult, workers) - for i := 0; i < workers; i++ { - res := <-resultCh - storeResults[res.idx] = g6StoreOperationResult{ - Worker: res.idx, - FileID: res.id, - } - if res.err != nil { - storeResults[res.idx].Error = res.err.Error() - } - if res.err != nil { - t.Fatalf("concurrent store worker %d failed: %v", res.idx, res.err) - } - ids[res.idx] = res.id + + holder.release(t) + fileID := storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, inPath) + verifyConcurrentInvariantsG6(t, dbconn, nil) + restoreMustMatchHashG6(t, dbconn, fileID, filepath.Join(restoreDir, "g6-contention-restored.bin"), fileHash) + }) + } +} + +func TestAdversarialG6KilledLeaseHolderReleasesRepository(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("killed-process repository Lease release proof is required on Linux") + } + testgate.RequireDB(t) + testgate.RequireLongRun(t) + + codec := getenvOrDefaultAdversarialG6("COLDKEEP_CODEC", "plain") + if codec != "plain" && codec != "aes-gcm" { + t.Fatalf("unsupported killed-holder proof codec %q", codec) + } + configureAdversarialG6Codec(t, codec) + + dbconn, env, repoRoot, binPath, tmp, _ := setupAdversarialG6Env(t) + defer dbconn.Close() + + inputDir := filepath.Join(tmp, "input") + restoreDir := filepath.Join(tmp, "restore") + if err := os.MkdirAll(inputDir, 0o755); err != nil { + t.Fatalf("mkdir input: %v", err) + } + if err := os.MkdirAll(restoreDir, 0o755); err != nil { + t.Fatalf("mkdir restore: %v", err) + } + + inPath := testutils.CreateTempFile(t, inputDir, "g6-killed-holder.bin", 256*1024+313) + fileHash := testutils.SHA256File(t, inPath) + holder := startG6RepositoryLeaseHolder(t, env) + + prepared, err := coordination.PrepareControlNamespace(env["COLDKEEP_STORAGE_DIR"]) + if err != nil { + t.Fatalf("prepare killed-holder control namespace: %v", err) + } + assertG6PersistentRepositoryLock(t, prepared.LockArtifactPath, "holder acquisition") + ownerBeforeKill := readG6OwnerMetadata(t, prepared.OwnerMetadataPath, "before holder kill") + if ownerBeforeKill.PID != holder.command.Process.Pid { + t.Fatalf("holder owner PID=%d want process PID=%d", ownerBeforeKill.PID, holder.command.Process.Pid) + } + if ownerBeforeKill.Operation != coordination.OperationStore || ownerBeforeKill.IdentityHash != prepared.Identity.Hash { + t.Fatalf("holder owner metadata=%+v want store owner for repository identity %s", ownerBeforeKill, prepared.Identity.Hash) + } + + busyResult := runColdkeepCommandWithTimeoutG6( + t, + repoRoot, + binPath, + env, + "store", "--codec", codec, inPath, "--output", "json", + ) + assertRepositoryBusyCLIResultG6(t, busyResult) + + var storedRows int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM logical_file WHERE file_hash = $1`, fileHash).Scan(&storedRows); err != nil { + t.Fatalf("count logical files after killed-holder Busy contender: %v", err) + } + if storedRows != 0 { + t.Fatalf("killed-holder Busy contender stored %d logical-file rows; want 0", storedRows) + } + + holder.kill(t) + assertG6PersistentRepositoryLock(t, prepared.LockArtifactPath, "holder death") + ownerAfterKill := readG6OwnerMetadata(t, prepared.OwnerMetadataPath, "after holder kill") + if ownerAfterKill != ownerBeforeKill { + t.Fatalf("stale owner metadata after holder kill=%+v want %+v", ownerAfterKill, ownerBeforeKill) + } + + fileID := storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, inPath) + assertG6PersistentRepositoryLock(t, prepared.LockArtifactPath, "successful reacquisition and release") + if _, err := os.Lstat(prepared.OwnerMetadataPath); !os.IsNotExist(err) { + t.Fatalf("owner metadata exists after successful reacquisition and release, stat err=%v", err) + } + verifyConcurrentInvariantsG6(t, dbconn, nil) + restoreMustMatchHashG6(t, dbconn, fileID, filepath.Join(restoreDir, "g6-killed-holder-restored.bin"), fileHash) +} + +func TestAdversarialG6LiveGCExcludesIndependentStoreProcess(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("live-GC independent-process repository barrier proof is required on Linux") + } + testgate.RequireDB(t) + testgate.RequireLongRun(t) + + codec := getenvOrDefaultAdversarialG6("COLDKEEP_CODEC", "plain") + if codec != "plain" && codec != "aes-gcm" { + t.Fatalf("unsupported live-GC barrier proof codec %q", codec) + } + configureAdversarialG6Codec(t, codec) + + dbconn, env, repoRoot, binPath, tmp, _ := setupAdversarialG6Env(t) + defer dbconn.Close() + // The server-observed relation wait is the synchronization mechanism. Give + // the child enough lock/statement budget for diagnostics on loaded runners. + env["COLDKEEP_DB_LOCK_TIMEOUT_MS"] = "30000" + env["COLDKEEP_DB_STATEMENT_TIMEOUT_MS"] = "30000" + + inputDir := filepath.Join(tmp, "input") + restoreDir := filepath.Join(tmp, "restore") + if err := os.MkdirAll(inputDir, 0o755); err != nil { + t.Fatalf("mkdir input: %v", err) + } + if err := os.MkdirAll(restoreDir, 0o755); err != nil { + t.Fatalf("mkdir restore: %v", err) + } + + anchorPath := testutils.CreateTempFile(t, inputDir, "g6-live-gc-anchor.bin", 256*1024+313) + anchorHash := testutils.SHA256File(t, anchorPath) + anchorID := storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, anchorPath) + contenderPath := testutils.CreateTempFile(t, inputDir, "g6-live-gc-contender.bin", 192*1024+197) + contenderHash := testutils.SHA256File(t, contenderPath) + + prepared, err := coordination.PrepareControlNamespace(env["COLDKEEP_STORAGE_DIR"]) + if err != nil { + t.Fatalf("prepare live-GC control namespace: %v", err) + } + assertG6PersistentRepositoryLock(t, prepared.LockArtifactPath, "anchor store") + + locker, err := dbconn.Conn(context.Background()) + if err != nil { + t.Fatalf("reserve live-GC barrier session: %v", err) + } + defer locker.Close() + observer, err := dbconn.Conn(context.Background()) + if err != nil { + t.Fatalf("reserve live-GC observer session: %v", err) + } + defer observer.Close() + + barrierTx, err := locker.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin live-GC relation barrier: %v", err) + } + barrierReleased := false + defer func() { + if !barrierReleased { + _ = barrierTx.Rollback() + } + }() + var blockerPID int + if err := barrierTx.QueryRow(`SELECT pg_backend_pid()`).Scan(&blockerPID); err != nil { + t.Fatalf("query live-GC barrier backend PID: %v", err) + } + if _, err := barrierTx.Exec(`LOCK TABLE physical_file IN ACCESS EXCLUSIVE MODE`); err != nil { + t.Fatalf("acquire live-GC physical_file relation barrier: %v", err) + } + + liveGC := startG6AsyncCLICommand(t, repoRoot, binPath, env, "gc", "--output", "json") + waiterPID := waitForG6GCPhysicalGraphLockWait(t, observer, blockerPID) + if waiterPID == blockerPID { + t.Fatalf("live-GC waiter PID=%d unexpectedly equals blocker PID", waiterPID) + } + + assertG6PersistentRepositoryLock(t, prepared.LockArtifactPath, "live GC server-observed wait") + owner := readG6OwnerMetadata(t, prepared.OwnerMetadataPath, "live GC server-observed wait") + if owner.PID != liveGC.command.Process.Pid { + t.Fatalf("live-GC owner PID=%d want process PID=%d", owner.PID, liveGC.command.Process.Pid) + } + if owner.Operation != coordination.OperationGarbageCollect || owner.IdentityHash != prepared.Identity.Hash { + t.Fatalf("live-GC owner metadata=%+v want gc owner for repository identity %s", owner, prepared.Identity.Hash) + } + + busyResult := runColdkeepCommandWithTimeoutG6( + t, + repoRoot, + binPath, + env, + "store", "--codec", codec, contenderPath, "--output", "json", + ) + assertRepositoryBusyCLIResultG6(t, busyResult) + + var storedRows int + if err := dbconn.QueryRow(`SELECT COUNT(*) FROM logical_file WHERE file_hash = $1`, contenderHash).Scan(&storedRows); err != nil { + t.Fatalf("count logical files after live-GC Busy contender: %v", err) + } + if storedRows != 0 { + t.Fatalf("live-GC Busy contender stored %d logical-file rows; want 0", storedRows) + } + + if err := barrierTx.Rollback(); err != nil { + t.Fatalf("release live-GC relation barrier: %v", err) + } + barrierReleased = true + gcResult := liveGC.result(t) + gcPayload := testutils.AssertCLIJSONOK(t, gcResult, "gc") + gcData := testutils.JSONMap(t, gcPayload, "data") + if dryRun, _ := gcData["dry_run"].(bool); dryRun { + t.Fatalf("live GC reported dry_run=true: %v", gcData) + } + if affected := testutils.JSONInt64(t, gcData, "affected_containers"); affected != 0 { + t.Fatalf("live GC affected containers=%d want 0", affected) + } + + assertG6PersistentRepositoryLock(t, prepared.LockArtifactPath, "live GC completion") + if _, err := os.Lstat(prepared.OwnerMetadataPath); !os.IsNotExist(err) { + t.Fatalf("owner metadata exists after live GC completion, stat err=%v", err) + } + + contenderID := storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, contenderPath) + verifyConcurrentInvariantsG6(t, dbconn, nil) + restoreMustMatchHashG6(t, dbconn, anchorID, filepath.Join(restoreDir, "g6-live-gc-anchor-restored.bin"), anchorHash) + restoreMustMatchHashG6(t, dbconn, contenderID, filepath.Join(restoreDir, "g6-live-gc-contender-restored.bin"), contenderHash) +} + +func assertG6PersistentRepositoryLock(t *testing.T, path, stage string) { + t.Helper() + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat repository.lock after %s: %v", stage, err) + } + if !info.Mode().IsRegular() { + t.Fatalf("repository.lock mode after %s=%v want regular", stage, info.Mode()) + } +} + +func readG6OwnerMetadata(t *testing.T, path, stage string) coordination.Owner { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read owner metadata %s: %v", stage, err) + } + owner, err := coordination.DecodeOwner(data) + if err != nil { + t.Fatalf("decode owner metadata %s: %v", stage, err) + } + return owner +} + +func TestAdversarialG6SequentialStoresSameFileConvergeDeterministically(t *testing.T) { + testgate.RequireDB(t) + testgate.RequireLongRun(t) + + for _, codec := range adversarialG6Codecs() { + t.Run(codec, func(t *testing.T) { + configureAdversarialG6Codec(t, codec) + + dbconn, env, repoRoot, binPath, tmp, _ := setupAdversarialG6Env(t) + defer dbconn.Close() + + inputDir := filepath.Join(tmp, "input") + restoreDir := filepath.Join(tmp, "restore") + if err := os.MkdirAll(inputDir, 0o755); err != nil { + t.Fatalf("mkdir input: %v", err) + } + if err := os.MkdirAll(restoreDir, 0o755); err != nil { + t.Fatalf("mkdir restore: %v", err) + } + + inPath := testutils.CreateTempFile(t, inputDir, "g6-same-file.bin", 2*1024*1024+313) + wantHash := testutils.SHA256File(t, inPath) + + const stores = 6 + ids := make([]int64, stores) + for i := range ids { + ids[i] = storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, inPath) } - verifyConcurrentInvariantsG6(t, dbconn, &g6FailureDiagnosticContext{ - TestName: t.Name(), - Backend: "postgres", - OuterJobCodec: outerJobCodec, - InnerSubtest: codec, - GOMAXPROCS: runtime.GOMAXPROCS(0), - Concurrency: workers, - IsolatedDB: testDBName, - TempRoot: tmp, - StoreResults: storeResults, - }) + verifyConcurrentInvariantsG6(t, dbconn, nil) baseGraph := testutils.QueryChunkGraph(t, dbconn, ids[0]) if len(baseGraph) == 0 { @@ -1112,7 +2138,7 @@ func TestAdversarialG6ConcurrentStoresSameFileConvergeDeterministically(t *testi } for j := range baseGraph { if baseGraph[j] != graph[j] { - t.Fatalf("chunk graph drift between concurrent stores at file=%d index=%d: base=%+v got=%+v", i, j, baseGraph[j], graph[j]) + t.Fatalf("chunk graph drift between sequential stores at file=%d index=%d: base=%+v got=%+v", i, j, baseGraph[j], graph[j]) } } } @@ -1141,7 +2167,7 @@ func TestAdversarialG6DeterministicStoreInterleavingPostgres(t *testing.T) { } } -func TestAdversarialG6ConcurrentStoresSharedChunkInputsPreserveHealthyRestores(t *testing.T) { +func TestAdversarialG6SequentialStoresSharedChunksPreserveHealthyRestores(t *testing.T) { testgate.RequireDB(t) testgate.RequireLongRun(t) @@ -1161,30 +2187,41 @@ func TestAdversarialG6ConcurrentStoresSharedChunkInputsPreserveHealthyRestores(t t.Fatalf("mkdir restore: %v", err) } - paths := testutils.CreateSampleDataset(t, inputDir) - hybridA := paths["hybrid_a.bin"] - hybridB := paths["hybrid_b.bin"] + sharedPrefix := make([]byte, chunk.MaxChunkSize) + for i := range sharedPrefix { + sharedPrefix[i] = byte((i*31 + 7) % 251) + } + tailA := make([]byte, 64*1024) + tailB := make([]byte, 64*1024) + for i := range tailA { + tailA[i] = byte((i*17 + 3) % 251) + tailB[i] = byte((i*29 + 11) % 251) + } + hybridA := filepath.Join(inputDir, "hybrid_a.bin") + hybridB := filepath.Join(inputDir, "hybrid_b.bin") + if err := os.WriteFile(hybridA, append(append([]byte{}, sharedPrefix...), tailA...), 0o600); err != nil { + t.Fatalf("write hybrid_a: %v", err) + } + if err := os.WriteFile(hybridB, append(append([]byte{}, sharedPrefix...), tailB...), 0o600); err != nil { + t.Fatalf("write hybrid_b: %v", err) + } hashA := testutils.SHA256File(t, hybridA) hashB := testutils.SHA256File(t, hybridB) - resultACh := make(chan error, 1) - resultBCh := make(chan error, 1) - var fileAID, fileBID int64 - go func() { - var err error - fileAID, err = storeFileWithCodecCLIG6Async(repoRoot, binPath, env, codec, hybridA) - resultACh <- err - }() - go func() { - var err error - fileBID, err = storeFileWithCodecCLIG6Async(repoRoot, binPath, env, codec, hybridB) - resultBCh <- err - }() - if errA := <-resultACh; errA != nil { - t.Fatalf("concurrent store hybrid_a failed: %v", errA) + fileAID := storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, hybridA) + fileBID := storeFileWithCodecCLIG6(t, repoRoot, binPath, env, codec, hybridB) + + var sharedChunks int + if err := dbconn.QueryRow(` + SELECT COUNT(DISTINCT fc_a.chunk_id) + FROM file_chunk fc_a + JOIN file_chunk fc_b ON fc_b.chunk_id = fc_a.chunk_id + WHERE fc_a.logical_file_id = $1 AND fc_b.logical_file_id = $2 + `, fileAID, fileBID).Scan(&sharedChunks); err != nil { + t.Fatalf("count shared chunks: %v", err) } - if errB := <-resultBCh; errB != nil { - t.Fatalf("concurrent store hybrid_b failed: %v", errB) + if sharedChunks == 0 { + t.Fatal("expected hybrid inputs to reference at least one shared chunk") } verifyConcurrentInvariantsG6(t, dbconn, nil) diff --git a/tests/adversarial/g6_diagnostics_test.go b/tests/adversarial/g6_diagnostics_test.go index 100d80c3..c164392f 100644 --- a/tests/adversarial/g6_diagnostics_test.go +++ b/tests/adversarial/g6_diagnostics_test.go @@ -53,9 +53,23 @@ func expectedG6DiagnosticManifest() g6ChunkFailureDiagnosticManifest { IsolatedDatabaseName: "coldkeep_adversarial_g6_123", OffendingChunkHash: "abc123", StoreResults: []g6StoreOperationResult{ - {Worker: 0, FileID: 1}, + { + Worker: 0, + FileID: 1, + LifecycleTrace: []string{"event=store_reuse_validation_failed file_id=1"}, + StartedUTC: time.Date(2026, 7, 21, 5, 41, 0, 0, time.UTC), + FinishedUTC: time.Date(2026, 7, 21, 5, 41, 1, 0, time.UTC), + }, {Worker: 1, Error: "boom"}, }, + PackedBlocks: []g6PackedBlockRecord{{ + BlockID: 7, + PhysicalHash: "expected-physical-hash", + ActualPhysicalHash: "actual-physical-hash", + Members: []g6PackedBlockMember{{ChunkID: 3, ChunkHash: "abc123"}}, + EncodedMembers: []g6EncodedBlockMember{{ChunkID: 3, Offset: 0, Size: 64}}, + }}, + PhysicalFiles: []g6PhysicalFileRecord{{ID: 9, Path: "/tmp/input", LogicalFileID: 1}}, } } @@ -105,6 +119,94 @@ func assertExpectedG6DiagnosticManifest(t *testing.T, manifest g6ChunkFailureDia if len(manifest.StoreResults) != 2 || manifest.StoreResults[1].Error != "boom" { t.Fatalf("unexpected store results: %+v", manifest.StoreResults) } + if len(manifest.StoreResults[0].LifecycleTrace) != 1 || !strings.Contains(manifest.StoreResults[0].LifecycleTrace[0], "store_reuse_validation_failed") { + t.Fatalf("unexpected lifecycle trace: %+v", manifest.StoreResults[0].LifecycleTrace) + } + if manifest.StoreResults[0].StartedUTC.IsZero() || !manifest.StoreResults[0].FinishedUTC.After(manifest.StoreResults[0].StartedUTC) { + t.Fatalf("unexpected store operation timestamps: %+v", manifest.StoreResults[0]) + } + if len(manifest.PackedBlocks) != 1 || len(manifest.PackedBlocks[0].Members) != 1 || manifest.PackedBlocks[0].Members[0].ChunkID != 3 || len(manifest.PackedBlocks[0].EncodedMembers) != 1 { + t.Fatalf("unexpected packed block diagnostics: %+v", manifest.PackedBlocks) + } + if len(manifest.PhysicalFiles) != 1 || manifest.PhysicalFiles[0].ID != 9 || manifest.PhysicalFiles[0].LogicalFileID != 1 { + t.Fatalf("unexpected physical-file diagnostics: %+v", manifest.PhysicalFiles) + } +} + +func TestG6ChunkIDPatternAcceptsVerifierFormats(t *testing.T) { + for _, tc := range []struct { + name string + input string + want string + }{ + {name: "space", input: "chunk 3 has both mappings", want: "3"}, + {name: "equals", input: "encoded entry missing chunk=17 offset=0", want: "17"}, + } { + t.Run(tc.name, func(t *testing.T) { + matches := g6ChunkIDPattern.FindStringSubmatch(tc.input) + if len(matches) != 2 || matches[1] != tc.want { + t.Fatalf("unexpected chunk match for %q: %v", tc.input, matches) + } + }) + } +} + +func TestFilterG6LifecycleTraceKeepsAllowedEventsAndRedactsSecrets(t *testing.T) { + env := map[string]string{ + "COLDKEEP_KEY": "secret-key", + "DB_PASSWORD": "secret-password", + } + output := strings.Join([]string{ + "ordinary CLI output secret-key", + "2026/07/21 event=store_reuse_validation_failed file_id=1 error=secret-key", + "2026/07/21 event=chunk_reuse_validation_failed chunk_id=3 error=secret-password", + "2026/07/21 event=store_chunk_reclaim action=write_rebuild chunk_id=3", + "unrelated event=restore_block_read action=start", + }, "\n") + + trace := filterG6LifecycleTrace(output, env) + if len(trace) != 3 { + t.Fatalf("expected three lifecycle events, got %d: %v", len(trace), trace) + } + joined := strings.Join(trace, "\n") + if containsSecretG6(joined) { + t.Fatalf("lifecycle trace leaked secret material: %s", joined) + } + if !strings.Contains(joined, "[REDACTED]") || strings.Contains(joined, "restore_block_read") { + t.Fatalf("unexpected filtered lifecycle trace: %s", joined) + } +} + +func TestAttachG6ActualPhysicalHashRejectsInvalidBounds(t *testing.T) { + for _, tc := range []struct { + name string + record g6PackedBlockRecord + }{ + { + name: "negative size", + record: g6PackedBlockRecord{StoredSize: -1}, + }, + { + name: "negative offset", + record: g6PackedBlockRecord{ContainerOffset: -1}, + }, + { + name: "past container maximum", + record: g6PackedBlockRecord{ + ContainerOffset: 90, + StoredSize: 11, + ContainerMaxSize: 100, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + record := tc.record + attachG6ActualPhysicalHash(&record) + if record.ActualPhysicalHashError == "" { + t.Fatal("expected invalid bounds to be recorded as a diagnostic error") + } + }) + } } func containsSecretG6(s string) bool {