From a2e0a2bafcbbbc2dda7150331296177b246abe1b Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 12:12:46 +0200 Subject: [PATCH 01/27] docs: start v1.13.12 engine completion train --- CHANGELOG.md | 7 ++- README.md | 19 +++--- docs/architecture/engine-boundary-plan.md | 21 ++++--- docs/release/v1.13/README.md | 19 +++--- docs/release/v1.13/v1.13.12-phase-list.md | 34 ++++++++++ ...ase0-post-v1.13.11-truth-reconciliation.md | 62 +++++++++++++++++++ docs/release/v1.13/v1.13.12-release-gate.md | 35 +++++++++++ docs/release/v1.13/v1.13.12-scope.md | 47 ++++++++++++++ .../v1.13/v1.13.12-validation-checklist.md | 42 +++++++++++++ docs/release/v1.13/v1.13.x-release-train.md | 20 +++--- 10 files changed, 271 insertions(+), 35 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase-list.md create mode 100644 docs/release/v1.13/v1.13.12-phase0-post-v1.13.11-truth-reconciliation.md create mode 100644 docs/release/v1.13/v1.13.12-release-gate.md create mode 100644 docs/release/v1.13/v1.13.12-scope.md create mode 100644 docs/release/v1.13/v1.13.12-validation-checklist.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd7b0bb..a71e9a0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,12 @@ project, do not start here; start with [README.md](README.md). ------------------------------------------------------------------------ -## v1.13.11 - 2026-08-18 — Safety and Backend Compatibility Gate Closure +## v1.13.11 - 2026-08-19 — Safety and Backend Compatibility Gate Closure + +- Published the stable v1.13.11 release from merge commit + `507859daccf25594142c61e5ab8209a751fb579a` after exact-head candidate, + hosted PR, main, tag, and release validation. This completes v1.13.11 but + does not close v1.x; v1.13.12 and v1.13.13 remain mandatory. - Completed the Phase 20 pre-release state transition and froze the exact-head candidate contract. The commit containing that contract must pass diff --git a/README.md b/README.md index aa9731d3..6c3492c7 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Coldkeep uses a visual identity based on an ice cube vault: ![CI](https://github.com/franchoy/coldkeep/actions/workflows/ci.yml/badge.svg) ![Go Version](https://img.shields.io/badge/go-1.25+-blue) ![License](https://img.shields.io/badge/license-Apache%202.0-blue) -![Status](https://img.shields.io/badge/status-v1.13.11%20ready%20for%20release-blue) +![Status](https://img.shields.io/badge/status-v1.13.12%20active-blue) ![Release](https://img.shields.io/github/v/release/franchoy/coldkeep?include_prereleases) > Status: v1.9 formalizes transform-based storage semantics (logical/compressed/physical layers) with block-level compression and explicit staged verification, while preserving deterministic restore, GC safety, snapshot semantics, and mixed-repository compatibility. @@ -35,14 +35,15 @@ a valid released baseline, but its earlier final-v1.x conclusion was superseded after release by a roadmap-to-code audit that identified remaining must-before-v2 work. -v1.13.11 has completed Phases 0–20 and is ready as a pre-release candidate for safety -and backend compatibility gate closure. One pull request is authorized only -after the immutable commit containing the final gate contract passes -candidate-head CI, Required Gate, CodeQL, and the complete clean local Profile -A gate. Merge, tag, and publication remain later operations. v1.13.12 and -v1.13.13 remain required follow-on releases; SQLite-first remains a future -local-product direction, PostgreSQL compatibility remains required, and v2.0 -implementation has not started. +v1.13.11 is complete, merged, tagged, and published. It closed the safety and +backend-compatibility gate at merge commit +`507859daccf25594142c61e5ab8209a751fb579a` without closing v1.x as a whole. + +v1.13.12 is the active release for mandatory engine and catalog completion. +v1.13.13 remains the required final independent v1.x and v2 handoff gate. +SQLite-default portable-repository productization remains a future v2.x +objective, PostgreSQL compatibility remains required, and v2.x implementation +is not authorized. coldkeep is a local-first content-addressed storage engine focused on deterministic restore, explicit integrity verification, and safe lifecycle behavior under failure scenarios. diff --git a/docs/architecture/engine-boundary-plan.md b/docs/architecture/engine-boundary-plan.md index 3ae52af7..ab2ee129 100644 --- a/docs/architecture/engine-boundary-plan.md +++ b/docs/architecture/engine-boundary-plan.md @@ -92,15 +92,22 @@ retained content, and snapshot restore retains explicit destination and selection semantics. This activation does not make every snapshot read-side workflow or the daemon/API contract complete. -## Current deferred boundaries +## Current mandatory completion boundaries -- `Engine.Store` remains single-file only; recursive/folder store is outside - the active route and returns `ErrNotImplemented` when requested there. +- `Engine.Store` remains single-file only; v1.13.12 must add a distinct + engine-owned folder/recursive store operation and remove the misleading + unsupported recursive request surface. - Snapshot list/show/stats/diff remain active but provisional mixed read-side - seams whose ownership and shape require an explicit v2.0 decision. -- Repair and recovery remain CLI/domain-owned corrective work; their - request/result types are candidate-only and neither is an active Engine - method. + seams. v1.13.12 must complete their engine ownership and neutral contracts. +- Repair, recovery, startup recovery, and doctor remain CLI/domain-owned + corrective work. v1.13.12 must activate explicit headless engine operations + and move correctness sequencing behind them. +- Snapshot graph, placement, restore-plan, and GC-plan catalog surfaces remain + unimplemented. v1.13.12 must implement and adopt all four in production. + +These items are mandatory v1.x architecture work. They are not v2.x +deferrals. SQLite-default productization and broader distributed coordination +remain the separate v2.x boundaries. ## Migration rule diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 61b69759..cd704b1e 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -76,8 +76,9 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree baseline, not the final v1.x release. - A post-release roadmap-to-code audit superseded the narrower final-v1.x conclusion and restored v1.13.11–v1.13.13 for remaining must-before-v2 work. -- `v1.13.11 — Safety and Backend Compatibility Gate Closure` is the single - release ready for pre-release validation. Phases 0–20 are complete. Phase 12 implements the +- `v1.13.11 — Safety and Backend Compatibility Gate Closure` is complete, + merged as `507859daccf25594142c61e5ab8209a751fb579a`, annotated-tagged, and + published. Phases 0–20 are complete. Phase 12 implements the repository-wide exclusive fail-fast Lease and proves native runtime plus the production Coordinator lifecycle on Linux, macOS, and Windows. Phase 13 preserves strengthened G6 integrity coverage and proves deterministic Linux @@ -144,15 +145,19 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree deterministic snapshot ordering. Selected schema/bootstrap/migration contracts and the G6 fail-closed remediation also have exact-head CI evidence, while broad backend parity remains intentionally unclaimed. - v1.13.11 is ready for exact-head pre-release validation; the latest released - version remains v1.13.10. Merge, tag, and publication remain later - operations. Its canonical trackers are + Its final exact-head evidence, PR #106 merge, main validation, annotated tag, + tag validation, stable publication, and release-branch deletion are + reconciled as completed release operations. Its canonical trackers are `v1.13.11-phase0-post-release-closure-correction-and-baseline.md`, `v1.13.11-scope.md`, `v1.13.11-phase-list.md`, `v1.13.11-validation-checklist.md`, and `v1.13.11-release-gate.md`. +- `v1.13.12 — Engine and Catalog Completion` is active from exact baseline + `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred + catalog planning APIs, production engine ownership, neutral contracts, and + enforceable thin-CLI boundary without changing storage semantics. - The updated `v1.13.x-release-train.md` is the authoritative current plan; - final v1.x completion is gated by v1.13.11–v1.13.13. v2.0 implementation has - not started. + final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an + audit-only final handoff gate, and v2.x implementation is not authorized. - `v1.13.10-engine-contract-documentation-truthfulness.md` records the current Engine contract boundary and its intentional limitations. - `v1.13.10-release-state-validator-contract.md` freezes the lifecycle, diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md new file mode 100644 index 00000000..36d81b41 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -0,0 +1,34 @@ +# Coldkeep v1.13.12 Phase List + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Status:** Active +**Branch:** `release/v1.13.12` + +| Phase | Title | Status | +|---:|---|---| +| 0 | Post-v1.13.11 Truth Reconciliation | Complete | +| 1 | Acceptance Inventory and Release Identity | Next | +| 2 | Engine Error and Neutrality Foundation | Not started | +| 3 | Stats, Inspect, Trace, and Verification Contracts | Not started | +| 4 | Catalog Contract Replacement | Not started | +| 5 | Snapshot Graph Implementation | Not started | +| 6 | Placement Implementation | Not started | +| 7 | Restore-Plan Implementation | Not started | +| 8 | Restore Production Adoption | Not started | +| 9 | GC-Plan Implementation and Adoption | Not started | +| 10 | Folder and Recursive Store Ownership | Not started | +| 11 | List and Search Ownership | Not started | +| 12 | Configuration Ownership | Not started | +| 13 | Snapshot Read-Side Completion | Not started | +| 14 | Inspect, Stats, and Verify Routing | Not started | +| 15 | Repair Activation | Not started | +| 16 | Recovery Activation | Not started | +| 17 | Doctor Activation | Not started | +| 18 | Application Composition and Thin CLI Closure | Not started | +| 19 | Dependency and Contract Hardening | Not started | +| 20 | Compatibility and Adversarial Regression | Not started | +| 21 | Exact-Head Candidate Release Gate | Not started | + +Phase 21 produces candidate-level PASS/FAIL and merge-authorization readiness +only. It cannot pre-claim merge, main, tag, publication, or branch-deletion +evidence. diff --git a/docs/release/v1.13/v1.13.12-phase0-post-v1.13.11-truth-reconciliation.md b/docs/release/v1.13/v1.13.12-phase0-post-v1.13.11-truth-reconciliation.md new file mode 100644 index 00000000..748495e8 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase0-post-v1.13.11-truth-reconciliation.md @@ -0,0 +1,62 @@ +# Coldkeep v1.13.12 Phase 0 — Post-v1.13.11 Truth Reconciliation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `0 — Post-v1.13.11 Truth Reconciliation` +**Status:** Complete +**Branch:** `release/v1.13.12` +**Starting baseline:** `507859daccf25594142c61e5ab8209a751fb579a` + +## Decision + +v1.13.11 is complete, merged, annotated-tagged, and published. v1.x is not +complete. The remaining blocker is the mandatory v1.12/v1.13 architecture +contract, not a newly discovered storage-safety or release-evidence failure. + +v1.13.12 is therefore the active release. v1.13.13 remains mandatory and is +reserved for the final independent audit and v2 handoff decision. v2.x work is +not authorized. + +## v1.13.11 provenance + +- Final PR head: `a2b903a12235c6d167ead38480333924629750da`. +- PR: `#106`, merged to `main` on 2026-08-19. +- Merge and v1.13.12 starting commit: + `507859daccf25594142c61e5ab8209a751fb579a`. +- Annotated tag: `v1.13.11`, peeling to the merge commit. +- Stable release: `Coldkeep v1.13.11 — Safety and Backend Compatibility Gate Closure`, + published 2026-08-19. +- `release/v1.13.11` is absent locally and from the fetched remote refs. + +The accepted final-head Profile A evidence remains rooted at +`/tmp/pr106-profile-a-a2b903a.bAI6iP/`. It is immutable historical evidence and +must not be executed or altered by v1.13.12. + +## Mandatory v1.13.12 work + +The authoritative release train requires v1.13.12 to: + +- implement and adopt catalog snapshot-graph, placement, restore-plan, and + GC-plan methods; +- complete engine-owned production orchestration, including folder store, + list/search, configuration, inspect, snapshot reads, repair, recovery, + startup recovery, and doctor; +- replace provisional or misleading internal contracts with neutral typed + request/result and error contracts; +- eliminate direct production CLI access to DB and lower execution layers; +- enforce the thin-CLI and dependency-direction boundary; and +- preserve storage semantics, public CLI/JSON/exit contracts, repository + format, schema, and dual-backend behavior. + +## Explicit v2.x boundaries + +SQLite-default portable-repository productization is deferred to v2.x while +SQLite/PostgreSQL contract parity remains mandatory. Cross-host, NAS, +network-filesystem, and distributed coordination are also later work. These +deferrals do not excuse any v1.13.12 engine/catalog acceptance criterion. + +## Phase authorization + +This phase changes documentation only. It creates no runtime, version, schema, +migration, storage, CI, or release-state behavior. Phase 1 is authorized to +activate the v1.13.12 identity and freeze the detailed ownership inventory and +compatibility baseline. diff --git a/docs/release/v1.13/v1.13.12-release-gate.md b/docs/release/v1.13/v1.13.12-release-gate.md new file mode 100644 index 00000000..d9527df6 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-release-gate.md @@ -0,0 +1,35 @@ +# Coldkeep v1.13.12 Release Gate + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Status:** Not run +**Branch:** `release/v1.13.12` + +## Candidate boundary + +The release gate applies to one frozen release-branch candidate SHA. Any +tracked candidate change invalidates all prior candidate-level gate evidence +and requires a complete restart. + +Required candidate evidence includes isolated plain and AES-GCM matrices, +SQLite/PostgreSQL contracts and round trips, process coordination, four +benchmark-integrity profiles and checksums, timing advisories, legacy, +cross-platform, snapshot, coverage, no-skip, static-analysis, storage +fingerprint, and clean-tree results. + +Local HEAD, origin release branch, PR head, Codacy, Required Gate, +CodeQL/security, and all hosted candidate checks must resolve to the same +candidate SHA. + +## Decision boundary + +This document may record only candidate `PASS` or `FAIL` and, after `PASS`, +merge-authorization readiness. It must not pre-claim merge, main CI, branch +deletion, tag, tag CI, stable publication, or v1.x closure. + +## Separate post-gate chain + +After explicit authorization: merge, validate exact main merge commit, delete +the release branch, create the annotated tag, validate tag CI, publish the +stable release, then reconcile each distinct identity and evidence set. + +v1.13.13 remains the sole final v1.x closure and v2 handoff gate. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md new file mode 100644 index 00000000..285a9944 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -0,0 +1,47 @@ +# Coldkeep v1.13.12 Scope + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Status:** Active +**Branch:** `release/v1.13.12` + +## Objective + +Complete the mandatory v1.x engine/catalog architecture without changing +storage semantics or public operator contracts. + +## Required outcomes + +- All four catalog planning methods are implemented, dual-backend tested, and + consumed by production behavior. +- All production repository operations cross an active neutral engine + request/result boundary. +- The CLI owns parsing, argument-file ingestion, compatibility projection, + rendering, prompts, outer coordination, and application composition only. +- Repair, recovery, startup recovery, and doctor have explicit headless engine + ownership. +- Direct CLI-to-DB/domain/storage execution is rejected by a static guard. +- v1.13.13 receives an audit-only surface with no promised-but-unimplemented + contract. + +## Frozen compatibility + +Do not change public CLI syntax, JSON shapes, exit codes, storage bytes, +repository format, schema/migrations, default-backend behavior, transaction or +lock semantics, failure ordering, or packed/legacy compatibility. + +Internal provisional engine and catalog APIs may change. SQLite and PostgreSQL +must remain supported. SQLite-default productization is not part of this +release. + +## Tooling exclusions + +Benchmark and determinism harnesses remain external tooling. Mutation +simulation must remain isolated from live repository state. Any simulation or +planning operation that reads a live repository must use the engine boundary. + +## Release discipline + +Every phase remains on `release/v1.13.12`, receives focused tests and one +reviewable commit, and preserves a clean tracked tree. Phase 21 validates one +frozen candidate SHA. Merge, main validation, branch deletion, annotated tag, +tag validation, and publication are later separately authorized operations. diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md new file mode 100644 index 00000000..9602ff67 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -0,0 +1,42 @@ +# Coldkeep v1.13.12 Validation Checklist + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Status:** Active +**Branch:** `release/v1.13.12` + +## Phase 0 + +- [x] Branch created from exact main commit + `507859daccf25594142c61e5ab8209a751fb579a`. +- [x] v1.13.11 marked merged, tagged, published, and complete. +- [x] v1.13.12 marked active and v1.13.13 preserved as mandatory. +- [x] v1.x remains explicitly incomplete. +- [x] SQLite-default and broader coordination remain v2.x deferrals. +- [x] Mandatory engine/catalog completion is not deferred to v2.x. + +## Per-phase gate + +- [ ] Exact expected branch and HEAD recorded. +- [ ] Tracked worktree clean before and after validation. +- [ ] Focused tests pass. +- [ ] Public compatibility fixtures remain unchanged. +- [ ] Repository storage fingerprint remains unchanged. +- [ ] One phase-scoped commit created. + +## Completion gate + +- [ ] Four catalog planning APIs implemented and used in production. +- [ ] Production operations route through active engine methods. +- [ ] Neutral DTO and typed-error contracts pass static checks. +- [ ] Thin-CLI AST/dependency guards pass with reviewed exclusions only. +- [ ] SQLite/PostgreSQL shared contract suites pass. +- [ ] Packed/legacy restore and GC parity passes. +- [ ] Repair/recovery/doctor fault and ordering suites pass. +- [ ] CLI, JSON, exit-code, schema, format, locking, and failure semantics pass. +- [ ] Exact-head Profile A and hosted candidate checks pass. +- [ ] Candidate PASS/FAIL and merge-authorization readiness recorded. + +All fresh validation must use a new named `/tmp` evidence root, isolated +per-stage storage and databases, scrubbed Coldkeep environment, and exact-name +PostgreSQL scratch database cleanup. A test that cannot satisfy those rules is +unavailable evidence, not a redirected shared-state test. diff --git a/docs/release/v1.13/v1.13.x-release-train.md b/docs/release/v1.13/v1.13.x-release-train.md index b461f336..a9ab8ab4 100644 --- a/docs/release/v1.13/v1.13.x-release-train.md +++ b/docs/release/v1.13/v1.13.x-release-train.md @@ -606,24 +606,22 @@ independent evidence. ### `v1.13.11 — Safety and Backend Compatibility Gate Closure` -**Status:** Ready for release; Phases 0–20 complete +**Status:** Released and operationally closed Primary scope: executable SQLite/PostgreSQL compatibility proof; same-host repository coordination; live-GC barrier proof; deterministic snapshot selection; bounded container reads and decompression; metadata consistency; JSON integer fidelity; targeted fail-closed SQL mutations; and required CI -evidence. Canonical trackers are the v1.13.11 Phase 0 baseline, scope, phase -list, validation checklist, and release gate. Phase 19 reconciled the aggregate -validation/evidence contracts and preserved their bounded proof claims without -product or CI-policy changes. Phase 20 freezes the prospective exact-head gate -contract and pre-release state. The immutable candidate must pass -candidate-head CI, Required Gate, CodeQL, and the complete clean local Profile -A gate before one pull request to `main` is authorized; merge, tag, and -publication remain later operations. +evidence. PR #106 merged final candidate +`a2b903a12235c6d167ead38480333924629750da` into `main` as +`507859daccf25594142c61e5ab8209a751fb579a` on 2026-08-19. The annotated +`v1.13.11` tag peels to that merge commit, and the stable GitHub release was +published on 2026-08-19. The release branch was deleted. These facts complete +v1.13.11 only; they do not supersede v1.13.12 or v1.13.13. ### `v1.13.12 — Engine and Catalog Completion` -**Status:** Planned after v1.13.11 +**Status:** Active; mandatory before v1.13.13 Implement deferred catalog methods, complete engine-owned orchestration, remove remaining user-facing direct DB and lower-layer CLI paths, complete thin-wrapper @@ -632,7 +630,7 @@ an explicit headless maintenance boundary. ### `v1.13.13 — Final v1.x and v2 Handoff Gate` -**Status:** Planned after v1.13.12 +**Status:** Mandatory after v1.13.12; not yet authorized Audit the expanded roadmap against runtime code; rerun correctness, race, adversarial, cross-platform, dual-backend, long-run, smoke, and benchmark From b33985595cdb517ee8f31e1bdd1ab1cb4a074a2d Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 12:19:02 +0200 Subject: [PATCH 02/27] release: activate v1.13.12 identity and inventory --- CHANGELOG.md | 9 ++ PRE_RELEASE_CHECKLIST.md | 4 +- docs/release/v1.13/README.md | 3 +- docs/release/v1.13/v1.13.11-release-gate.md | 14 +- docs/release/v1.13/v1.13.11-scope.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 118 ++++++++++++--- ...ceptance-inventory-and-release-identity.md | 87 +++++++++++ ....13.12-release-state-validator-contract.md | 17 +++ .../v1.13.12-release-train-reconciliation.md | 14 ++ docs/release/v1.13/v1.13.12-scope.md | 5 + .../v1.13/v1.13.12-validation-checklist.md | 141 ++++++++++++++---- internal/version/version.go | 2 +- internal/version/version_test.go | 2 +- 13 files changed, 355 insertions(+), 63 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase1-acceptance-inventory-and-release-identity.md create mode 100644 docs/release/v1.13/v1.13.12-release-state-validator-contract.md create mode 100644 docs/release/v1.13/v1.13.12-release-train-reconciliation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a71e9a0d..70ac21c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,15 @@ project, do not start here; start with [README.md](README.md). ------------------------------------------------------------------------ +## v1.13.12 - Unreleased — Engine and Catalog Completion + +- Started the mandatory architecture-completion release from exact v1.13.11 + merge commit `507859daccf25594142c61e5ab8209a751fb579a`. +- Activated version 1.13.12 and froze the production ownership, compatibility, + catalog-adoption, and thin-CLI acceptance baseline. + +------------------------------------------------------------------------ + ## v1.13.11 - 2026-08-19 — Safety and Backend Compatibility Gate Closure - Published the stable v1.13.11 release from merge commit diff --git a/PRE_RELEASE_CHECKLIST.md b/PRE_RELEASE_CHECKLIST.md index 9fa7a5c0..0642a4cd 100644 --- a/PRE_RELEASE_CHECKLIST.md +++ b/PRE_RELEASE_CHECKLIST.md @@ -286,7 +286,7 @@ python3 scripts/validate_release_state.py --state auto go build -o coldkeep ./cmd/coldkeep -expected_version="1.13.11" +expected_version="1.13.12" human_version=$(./coldkeep version) if [ "$human_version" != "coldkeep version $expected_version" ]; then @@ -314,7 +314,7 @@ fi Expected: local quality checks match CI intent and produce no diff or lint/format failures. -Expected: the built CLI reports exactly 1.13.11 in both human and JSON modes. +Expected: the built CLI reports exactly 1.13.12 in both human and JSON modes. A version mismatch blocks Profile A and release approval. Note: `scripts/clean_test_storage.sh` removes `./storage`, `.ci-storage`, and diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index cd704b1e..25a64ce0 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -154,7 +154,8 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree - `v1.13.12 — Engine and Catalog Completion` is active from exact baseline `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and - enforceable thin-CLI boundary without changing storage semantics. + enforceable thin-CLI boundary without changing storage semantics. Phase 1 + is complete; Phase 2 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.11-release-gate.md b/docs/release/v1.13/v1.13.11-release-gate.md index be4beef1..1d748995 100644 --- a/docs/release/v1.13/v1.13.11-release-gate.md +++ b/docs/release/v1.13/v1.13.11-release-gate.md @@ -1,11 +1,21 @@ # Coldkeep v1.13.11 Release Gate **Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` -**Status:** Passed — candidate preparation only; exact-head execution required +**Status:** Passed and released **Branch:** `release/v1.13.11` **Starting baseline:** `423c57815580c39bee4f79ecd81570e9cfa9d273` **Source version:** `1.13.11` -**Gate status:** Ready for exact-head execution +**Gate status:** Complete + +## Final verdict + +`PASS — v1.13.11 RELEASED` + +Final candidate `a2b903a12235c6d167ead38480333924629750da` passed the +accepted exact-head Profile A and hosted candidate gates. PR #106 merged it as +`507859daccf25594142c61e5ab8209a751fb579a`; main validation, annotated tag +`v1.13.11`, tag validation, stable publication, and release-branch deletion +were separately completed. This verdict closes v1.13.11 only, not v1.x. ## Exact candidate contract diff --git a/docs/release/v1.13/v1.13.11-scope.md b/docs/release/v1.13/v1.13.11-scope.md index 2aebadcd..2dfacf36 100644 --- a/docs/release/v1.13/v1.13.11-scope.md +++ b/docs/release/v1.13/v1.13.11-scope.md @@ -1,7 +1,7 @@ # Coldkeep v1.13.11 Scope **Release:** `v1.13.11 — Safety and Backend Compatibility Gate Closure` -**Status:** Ready for release +**Status:** Released and operationally closed **Branch:** `release/v1.13.11` **Starting baseline:** `423c57815580c39bee4f79ecd81570e9cfa9d273` diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 36d81b41..45423d66 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -4,30 +4,100 @@ **Status:** Active **Branch:** `release/v1.13.12` -| Phase | Title | Status | -|---:|---|---| -| 0 | Post-v1.13.11 Truth Reconciliation | Complete | -| 1 | Acceptance Inventory and Release Identity | Next | -| 2 | Engine Error and Neutrality Foundation | Not started | -| 3 | Stats, Inspect, Trace, and Verification Contracts | Not started | -| 4 | Catalog Contract Replacement | Not started | -| 5 | Snapshot Graph Implementation | Not started | -| 6 | Placement Implementation | Not started | -| 7 | Restore-Plan Implementation | Not started | -| 8 | Restore Production Adoption | Not started | -| 9 | GC-Plan Implementation and Adoption | Not started | -| 10 | Folder and Recursive Store Ownership | Not started | -| 11 | List and Search Ownership | Not started | -| 12 | Configuration Ownership | Not started | -| 13 | Snapshot Read-Side Completion | Not started | -| 14 | Inspect, Stats, and Verify Routing | Not started | -| 15 | Repair Activation | Not started | -| 16 | Recovery Activation | Not started | -| 17 | Doctor Activation | Not started | -| 18 | Application Composition and Thin CLI Closure | Not started | -| 19 | Dependency and Contract Hardening | Not started | -| 20 | Compatibility and Adversarial Regression | Not started | -| 21 | Exact-Head Candidate Release Gate | Not started | +## Phase 0 — Post-v1.13.11 Truth Reconciliation + +**Status:** Complete + +- Evidence: [Phase 0 truth reconciliation](v1.13.12-phase0-post-v1.13.11-truth-reconciliation.md). + +## Phase 1 — Acceptance Inventory and Release Identity + +**Status:** Complete + +- Evidence: [Phase 1 identity and inventory](v1.13.12-phase1-acceptance-inventory-and-release-identity.md). +- Source, version test, release checklist, changelog, trackers, and release + validator now identify active version 1.13.12. +- Production ownership and compatibility baselines are frozen before routing. + +## Phase 2 — Engine Error and Neutrality Foundation + +**Status:** Next + +## Phase 3 — Stats, Inspect, Trace, and Verification Contracts + +**Status:** Not started + +## Phase 4 — Catalog Contract Replacement + +**Status:** Not started + +## Phase 5 — Snapshot Graph Implementation + +**Status:** Not started + +## Phase 6 — Placement Implementation + +**Status:** Not started + +## Phase 7 — Restore-Plan Implementation + +**Status:** Not started + +## Phase 8 — Restore Production Adoption + +**Status:** Not started + +## Phase 9 — GC-Plan Implementation and Adoption + +**Status:** Not started + +## Phase 10 — Folder and Recursive Store Ownership + +**Status:** Not started + +## Phase 11 — List and Search Ownership + +**Status:** Not started + +## Phase 12 — Configuration Ownership + +**Status:** Not started + +## Phase 13 — Snapshot Read-Side Completion + +**Status:** Not started + +## Phase 14 — Inspect, Stats, and Verify Routing + +**Status:** Not started + +## Phase 15 — Repair Activation + +**Status:** Not started + +## Phase 16 — Recovery Activation + +**Status:** Not started + +## Phase 17 — Doctor Activation + +**Status:** Not started + +## Phase 18 — Application Composition and Thin CLI Closure + +**Status:** Not started + +## Phase 19 — Dependency and Contract Hardening + +**Status:** Not started + +## Phase 20 — Compatibility and Adversarial Regression + +**Status:** Not started + +## Phase 21 — Exact-Head Candidate Release Gate + +**Status:** Not started Phase 21 produces candidate-level PASS/FAIL and merge-authorization readiness only. It cannot pre-claim merge, main, tag, publication, or branch-deletion diff --git a/docs/release/v1.13/v1.13.12-phase1-acceptance-inventory-and-release-identity.md b/docs/release/v1.13/v1.13.12-phase1-acceptance-inventory-and-release-identity.md new file mode 100644 index 00000000..8a5fdeeb --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase1-acceptance-inventory-and-release-identity.md @@ -0,0 +1,87 @@ +# Coldkeep v1.13.12 Phase 1 — Acceptance Inventory and Release Identity + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `1 — Acceptance Inventory and Release Identity` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Identity activation + +The executable, version test, reusable pre-release checklist, changelog, root +README, release README, active trackers, and release-state validator inputs now +identify 1.13.12 as the active development release. The v1.13.11 tag and +benchmark reference remain immutable historical identities. + +## Production ownership baseline + +This inventory is the pre-refactor source of truth. A path is complete only +when its production execution crosses the target engine/catalog boundary; an +interface or test-only adapter is insufficient. + +| Operation | Current production ownership or violation | Required closure | +|---|---|---| +| store file | Engine entry exists; CLI still constructs storage context and projects/finalizes lower-layer results | Preserve engine route; composition and finalization close in Phases 10/18 | +| store folder | CLI calls storage folder orchestration directly | Distinct `Engine.StoreFolder` in Phase 10 | +| restore | Engine routes IDs/stored paths, but CLI retains lower metadata/context seams | Catalog restore plan and complete engine route in Phases 7–8/18 | +| remove | Engine routes IDs/stored paths; CLI retains composition/adapters | Preserve semantics; close composition in Phase 18 | +| GC | Engine mutation route exists; catalog GC planning remains absent | Catalog plan production adoption in Phase 9 | +| stats | Engine route exists but request/result expose observability contracts and trace sink | Neutral DTOs in Phase 3; final CLI route proof in Phase 14 | +| inspect | Production CLI calls observability service directly | Neutral engine DTO and sole engine route in Phases 3/14 | +| verify | Engine executes verification; CLI queries summary details directly | Engine-owned complete result in Phases 3/14 | +| list/search | CLI opens DB and calls raw CLI-argument-aware listing helpers | Typed catalog queries and engine operations in Phase 11 | +| configuration | CLI constructs repository, validates values, and reads/writes metadata | Typed catalog and engine configuration operations in Phase 12 | +| snapshot list/tree | Engine list adapter exists; CLI constructs graph/tree semantics | Catalog graph plus engine-neutral graph in Phases 5/13 | +| snapshot show/stats | Mixed engine metadata and direct snapshot-domain file/stat paths | One engine operation per command in Phase 13 | +| snapshot diff | Summary uses engine; detailed/filter paths call snapshot domain directly | All modes and selectors engine-owned in Phase 13 | +| snapshot create/delete/restore | Engine mutations exist; delete preview and DB/context seams remain mixed | Preserve mutations; close remaining read/composition seams in Phases 13/18 | +| repair | CLI validates, deduplicates, sequences, and calls maintenance primitives | Active `Engine.Repair` in Phase 15 | +| recovery/startup | CLI calls recovery directly and owns startup recovery execution | Active `Engine.Recover` for both paths in Phase 16 | +| doctor | CLI sequences recovery, schema, verify, and audit | Composite `Engine.Doctor` in Phase 17 | +| live GC simulation | CLI calls observability planning against repository state | Route live planning through engine by Phase 18 | +| application setup | CLI directly opens DB/storage contexts and constructs engines repeatedly | Single application composition boundary in Phase 18 | + +## Catalog baseline + +`LoadSnapshotGraph`, `LoadChunkPlacements`, `LoadRestorePlanMetadata`, and +`LoadGCPlanMetadata` return `ErrNotImplemented`. Their existing DTOs cannot +fully represent production restore and planning semantics. Phases 4–9 must +replace those provisional shapes, implement SQLite/PostgreSQL behavior, prove +parity, and adopt each method in production. + +The production restore recipe currently includes ordered chunk identity and +hashes, legacy offsets/sizes/codecs/formats/nonces, packed block references and +compression metadata, container filenames and maxima, transaction lifetime, +and chunk pinning. The catalog replacement may not discard or reorder any of +those semantics. + +## Frozen compatibility evidence + +The following existing suites and fixtures form the minimum pre-change +compatibility baseline: + +- command parser, JSON envelope, error/exit, and boundary-preservation tests in + `cmd/coldkeep`; +- engine route and dual-backend contract tests in `internal/engine`; +- catalog backend contracts in `internal/catalog`; +- restore, packed/legacy, snapshot, GC, repair, recovery, verification, and + coordination suites under `internal`; +- `scripts/validate_release_state.py`, its 48-fixture test suite, and the + local CI-enforcement audit; and +- the existing benchmark-integrity fixtures and paired reference approved by + trusted tag v1.13.11. + +Refactoring may add stronger fixtures but may not weaken, delete, rename away, +or silently bypass existing compatibility assertions. + +## Tooling classification + +Benchmark and determinism commands are external diagnostic tooling and are not +production repository operations. Simulated mutation remains tooling only when +its storage is explicitly isolated. Any command that reads or plans against a +live repository—including GC simulation—must cross the engine boundary. + +## Phase 2 authorization + +Phase 2 may add the typed engine error model and strengthen neutral-contract +reflection/static tests. It may not migrate command behavior, alter public +errors or exits, change schema/storage, or begin catalog adoption. diff --git a/docs/release/v1.13/v1.13.12-release-state-validator-contract.md b/docs/release/v1.13/v1.13.12-release-state-validator-contract.md new file mode 100644 index 00000000..a66ef8e6 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-release-state-validator-contract.md @@ -0,0 +1,17 @@ +# Coldkeep v1.13.12 Release-State Validator Contract + +v1.13.12 reuses the fail-closed release-state validator introduced in +v1.13.10. During development it must resolve one active 1.13.12 identity across +source constants, version test, pre-release checklist, changelog, root/release +READMEs, scope, phase list, validation checklist, and the nonhistorical release +train. + +The required development topology is contiguous Phases 0–21 with completed +phases followed by exactly one `Next` phase and then only `Not started` phases. +Completed phases must reference existing evidence artifacts. The active Git +branch must be `release/v1.13.12`, and no `v1.13.12` tag may exist before the +separate tag operation. + +Candidate, merge, tag, and released states remain distinct. Phase 21 can +establish candidate PASS and merge readiness only; it cannot satisfy later +identity checks through prose. diff --git a/docs/release/v1.13/v1.13.12-release-train-reconciliation.md b/docs/release/v1.13/v1.13.12-release-train-reconciliation.md new file mode 100644 index 00000000..f5165593 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-release-train-reconciliation.md @@ -0,0 +1,14 @@ +# Coldkeep v1.13.12 Release Train Reconciliation + +The sole current authority is +[the v1.13.x release train](v1.13.x-release-train.md). + +- v1.13.11 is released and operationally closed at merge commit + `507859daccf25594142c61e5ab8209a751fb579a`. +- v1.13.12 is active and owns mandatory engine/catalog completion. +- v1.13.13 remains mandatory and audit-only. +- v1.x is not complete, and v2.x implementation is not authorized. + +Historical proposals below the release train's explicit historical boundary +remain evidence only. They cannot defer v1.13.12 requirements or authorize +v2.x work. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 285a9944..14a3efa4 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -9,6 +9,11 @@ Complete the mandatory v1.x engine/catalog architecture without changing storage semantics or public operator contracts. +## Current phase + +Phase 1 is Complete. Phase 2 is Next: establish the typed engine error model +and enforce neutral request/result contracts before migrating operations. + ## Required outcomes - All four catalog planning methods are implemented, dual-backend tested, and diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 9602ff67..669e0c50 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -4,39 +4,118 @@ **Status:** Active **Branch:** `release/v1.13.12` -## Phase 0 +## Global controls -- [x] Branch created from exact main commit +- [x] Exact starting commit is `507859daccf25594142c61e5ab8209a751fb579a`. -- [x] v1.13.11 marked merged, tagged, published, and complete. -- [x] v1.13.12 marked active and v1.13.13 preserved as mandatory. +- [x] No preliminary mutation occurred on `main`. +- [x] One phase-scoped commit at a time. +- [x] Public CLI/JSON/exit/storage/schema/default-backend behavior is frozen. +- [x] SQLite-default and broader coordination remain outside v1.13.12. +- [x] Local candidate and post-gate release evidence remain distinct. + +## Phase 0 — Post-v1.13.11 Truth Reconciliation + +**Phase status:** Complete + +- [x] v1.13.11 is recorded as merged, tagged, published, and complete. +- [x] v1.13.12 is active and v1.13.13 remains mandatory. - [x] v1.x remains explicitly incomplete. -- [x] SQLite-default and broader coordination remain v2.x deferrals. -- [x] Mandatory engine/catalog completion is not deferred to v2.x. - -## Per-phase gate - -- [ ] Exact expected branch and HEAD recorded. -- [ ] Tracked worktree clean before and after validation. -- [ ] Focused tests pass. -- [ ] Public compatibility fixtures remain unchanged. -- [ ] Repository storage fingerprint remains unchanged. -- [ ] One phase-scoped commit created. - -## Completion gate - -- [ ] Four catalog planning APIs implemented and used in production. -- [ ] Production operations route through active engine methods. -- [ ] Neutral DTO and typed-error contracts pass static checks. -- [ ] Thin-CLI AST/dependency guards pass with reviewed exclusions only. -- [ ] SQLite/PostgreSQL shared contract suites pass. -- [ ] Packed/legacy restore and GC parity passes. -- [ ] Repair/recovery/doctor fault and ordering suites pass. -- [ ] CLI, JSON, exit-code, schema, format, locking, and failure semantics pass. -- [ ] Exact-head Profile A and hosted candidate checks pass. -- [ ] Candidate PASS/FAIL and merge-authorization readiness recorded. - -All fresh validation must use a new named `/tmp` evidence root, isolated -per-stage storage and databases, scrubbed Coldkeep environment, and exact-name +- [x] Mandatory engine/catalog work is not deferred to v2.x. + +## Phase 1 — Acceptance Inventory and Release Identity + +**Phase status:** Complete + +- [x] Source, version test, and reusable checklist identify 1.13.12. +- [x] Active changelog and tracker identity agree. +- [x] Production ownership and direct-coupling inventory is frozen. +- [x] Compatibility fixture baseline and tooling exclusions are frozen. +- [x] Human/JSON version and release-state validation pass. + +## Phase 2 — Engine Error and Neutrality Foundation + +**Phase status:** Next + +## Phase 3 — Stats, Inspect, Trace, and Verification Contracts + +**Phase status:** Not started + +## Phase 4 — Catalog Contract Replacement + +**Phase status:** Not started + +## Phase 5 — Snapshot Graph Implementation + +**Phase status:** Not started + +## Phase 6 — Placement Implementation + +**Phase status:** Not started + +## Phase 7 — Restore-Plan Implementation + +**Phase status:** Not started + +## Phase 8 — Restore Production Adoption + +**Phase status:** Not started + +## Phase 9 — GC-Plan Implementation and Adoption + +**Phase status:** Not started + +## Phase 10 — Folder and Recursive Store Ownership + +**Phase status:** Not started + +## Phase 11 — List and Search Ownership + +**Phase status:** Not started + +## Phase 12 — Configuration Ownership + +**Phase status:** Not started + +## Phase 13 — Snapshot Read-Side Completion + +**Phase status:** Not started + +## Phase 14 — Inspect, Stats, and Verify Routing + +**Phase status:** Not started + +## Phase 15 — Repair Activation + +**Phase status:** Not started + +## Phase 16 — Recovery Activation + +**Phase status:** Not started + +## Phase 17 — Doctor Activation + +**Phase status:** Not started + +## Phase 18 — Application Composition and Thin CLI Closure + +**Phase status:** Not started + +## Phase 19 — Dependency and Contract Hardening + +**Phase status:** Not started + +## Phase 20 — Compatibility and Adversarial Regression + +**Phase status:** Not started + +## Phase 21 — Exact-Head Candidate Release Gate + +**Phase status:** Not started + +## Validation isolation + +Every fresh run uses a new named `/tmp` evidence root, isolated per-stage +storage and databases, scrubbed Coldkeep environment, and exact-name PostgreSQL scratch database cleanup. A test that cannot satisfy those rules is unavailable evidence, not a redirected shared-state test. diff --git a/internal/version/version.go b/internal/version/version.go index fc811c01..f76b3ae0 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -5,7 +5,7 @@ import "fmt" const ( Major = 1 Minor = 13 - Patch = 11 + Patch = 12 ) func String() string { diff --git a/internal/version/version_test.go b/internal/version/version_test.go index 35fa1eac..c6f897e1 100644 --- a/internal/version/version_test.go +++ b/internal/version/version_test.go @@ -3,7 +3,7 @@ package version import "testing" func TestStringReturnsSemverFromConstants(t *testing.T) { - if got, want := String(), "1.13.11"; got != want { + if got, want := String(), "1.13.12"; got != want { t.Fatalf("String() mismatch: got=%q want=%q", got, want) } } From 87f74b504ea84a6af86c6e43c4d817efaf6c7167 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 12:23:45 +0200 Subject: [PATCH 03/27] engine: add typed error and neutrality foundation --- CHANGELOG.md | 4 + docs/release/v1.13/README.md | 4 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...-engine-error-and-neutrality-foundation.md | 50 +++++ docs/release/v1.13/v1.13.12-scope.md | 5 +- .../v1.13/v1.13.12-validation-checklist.md | 12 +- internal/engine/contracts_test.go | 44 +++++ internal/engine/engine.go | 9 +- internal/engine/errors.go | 185 +++++++++++++++++- internal/engine/errors_test.go | 102 ++++++++++ 10 files changed, 396 insertions(+), 25 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ac21c2..321eb3dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ project, do not start here; start with [README.md](README.md). merge commit `507859daccf25594142c61e5ab8209a751fb579a`. - Activated version 1.13.12 and froze the production ownership, compatibility, catalog-adoption, and thin-CLI acceptance baseline. +- Added the stable backend-neutral engine error taxonomy, deterministic + translation helpers, invariant/cancellation preservation, and exhaustive + active-contract neutrality coverage with only the four explicit Phase 3 + observability DTO debts temporarily allowlisted. ------------------------------------------------------------------------ diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 25a64ce0..d077a5aa 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -154,8 +154,8 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree - `v1.13.12 — Engine and Catalog Completion` is active from exact baseline `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and - enforceable thin-CLI boundary without changing storage semantics. Phase 1 - is complete; Phase 2 is next. + enforceable thin-CLI boundary without changing storage semantics. Phases + 0–2 are complete; Phase 3 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 45423d66..129f97e6 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -21,11 +21,13 @@ ## Phase 2 — Engine Error and Neutrality Foundation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 2 error and neutrality foundation](v1.13.12-phase2-engine-error-and-neutrality-foundation.md). ## Phase 3 — Stats, Inspect, Trace, and Verification Contracts -**Status:** Not started +**Status:** Next ## Phase 4 — Catalog Contract Replacement diff --git a/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md b/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md new file mode 100644 index 00000000..f4f7f879 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md @@ -0,0 +1,50 @@ +# Coldkeep v1.13.12 Phase 2 — Engine Error and Neutrality Foundation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `2 — Engine Error and Neutrality Foundation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Implemented contract + +The engine now exposes stable codes for invalid argument, not found, +unsupported, invariant violation, verification failure, recovery failure, +conflict, cancellation, and ordinary operation failure. `engine.Error` exports +only neutral scalar classification fields; its wrapped cause is private and +remains available through the standard Go error chain. + +`TranslateError` deterministically recognizes cancellation/deadline, +`ErrNotImplemented`, and typed invariant errors while preserving exact human +messages. `TranslateErrorAs` permits operation-specific semantic mapping +without overriding universal cancellation or invariant classification. +`CodeOf`, `IsCode`, and the compatibility `IsUnsupported` helper provide +stable inspection without string parsing. + +No active operation was rerouted and no CLI error, JSON, exit, schema, storage, +or backend behavior changed in this phase. + +## Neutrality enforcement + +The structural neutrality walk now covers the typed engine error and active +Verify contracts. A separate reflection guard enumerates every active Engine +request/result and rejects an unreviewed omission. + +The only temporary exceptions are `StatsRequest`, `StatsResult`, +`InspectRequest`, and `InspectResult`, whose exact observability dependencies +are the explicit Phase 3 work. Adding another method or contract cannot expand +that exception implicitly. Phase 3 must remove the exception after replacing +those DTOs. + +## Validation + +- `go test -count=1 ./internal/engine`: pass. +- Release-state validator: development state, 1.13.12, zero violations before + the phase commit. +- Repository storage fingerprint before/after: + `8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. +- Validation state was isolated under + `/tmp/coldkeep-v11312-phase2.ZHJaqu/`. + +Phase 3 is authorized to replace the four observability-backed DTOs, add +neutral trace and exact dynamic-value representations, and enrich Verify +results. It must preserve CLI rendering and may not reroute commands yet. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 14a3efa4..df657895 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,8 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phase 1 is Complete. Phase 2 is Next: establish the typed engine error model -and enforce neutral request/result contracts before migrating operations. +Phases 0–2 are Complete. Phase 3 is Next: replace observability-backed stats, +inspect, trace, and verification shapes with engine-neutral DTOs before any +CLI routing changes. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 669e0c50..0e2bb4d5 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -35,11 +35,19 @@ ## Phase 2 — Engine Error and Neutrality Foundation -**Phase status:** Next +**Phase status:** Complete + +- [x] Nine stable engine error codes are defined. +- [x] Typed errors preserve exact messages and `errors.Is`/`errors.As` chains. +- [x] Cancellation, unsupported, invariant, and ordinary failures translate + deterministically. +- [x] Active contract coverage rejects unreviewed neutrality omissions. +- [x] Only the four exact Phase 3 observability-backed DTOs are temporarily + allowlisted. ## Phase 3 — Stats, Inspect, Trace, and Verification Contracts -**Phase status:** Not started +**Phase status:** Next ## Phase 4 — Catalog Contract Replacement diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 88e8d0db..10c8ddbd 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -33,6 +33,7 @@ func activeCoreContractTypes() []struct { val any }{ {"OperationWarning", engine.OperationWarning{}}, + {"Error", engine.Error{}}, {"BatchSummary", engine.BatchSummary{}}, {"SnapshotQuery", engine.SnapshotQuery{}}, {"StoreRequest", engine.StoreRequest{}}, @@ -50,6 +51,8 @@ func activeCoreContractTypes() []struct { {"RemoveStoredPathsResult", engine.RemoveStoredPathsResult{}}, {"GarbageCollectRequest", engine.GarbageCollectRequest{}}, {"GarbageCollectResult", engine.GarbageCollectResult{}}, + {"VerifyRequest", engine.VerifyRequest{}}, + {"VerifyResult", engine.VerifyResult{}}, {"SnapshotMeta", engine.SnapshotMeta{}}, {"SnapshotListRequest", engine.SnapshotListRequest{}}, {"SnapshotListResult", engine.SnapshotListResult{}}, @@ -65,6 +68,47 @@ func activeCoreContractTypes() []struct { } } +// TestActiveEngineContractNeutralityCoverage prevents a request/result pair +// from escaping the structural neutrality walk when a method is added. The +// four exact observability-backed DTOs are the frozen Phase 3 migration debt; +// no other omission is permitted, and Phase 3 must remove this allowlist. +func TestActiveEngineContractNeutralityCoverage(t *testing.T) { + covered := make(map[reflect.Type]bool) + for _, tc := range allEngineContractTypes() { + covered[reflect.TypeOf(tc.val)] = true + } + legacyPhase3 := map[reflect.Type]bool{ + reflect.TypeOf(engine.StatsRequest{}): true, + reflect.TypeOf(engine.StatsResult{}): true, + reflect.TypeOf(engine.InspectRequest{}): true, + reflect.TypeOf(engine.InspectResult{}): true, + } + + interfaceType := reflect.TypeOf((*engine.Engine)(nil)).Elem() + seenLegacy := make(map[reflect.Type]bool) + for i := 0; i < interfaceType.NumMethod(); i++ { + method := interfaceType.Method(i) + if method.Type.NumIn() != 2 || method.Type.NumOut() != 2 { + t.Fatalf("Engine.%s has unexpected signature %s", method.Name, method.Type) + } + for _, contractType := range []reflect.Type{method.Type.In(1), method.Type.Out(0)} { + if covered[contractType] { + continue + } + if legacyPhase3[contractType] { + seenLegacy[contractType] = true + continue + } + t.Errorf("Engine.%s contract %s is absent from the neutrality walk", method.Name, contractType) + } + } + for contractType := range legacyPhase3 { + if !seenLegacy[contractType] { + t.Errorf("stale Phase 3 neutrality exception for %s", contractType) + } + } +} + func activeSnapshotMutationContractTypes() []struct { name string val any diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 2f68f26d..5b90d9f0 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -24,14 +24,7 @@ // domain packages must not import internal/engine package engine -import ( - "context" - "errors" -) - -// ErrNotImplemented is returned by active engine methods when a genuinely -// deferred mode remains outside the implemented surface. -var ErrNotImplemented = errors.New("engine operation not implemented") +import "context" // Engine is the behavior-preserving facade for coldkeep operations. // diff --git a/internal/engine/errors.go b/internal/engine/errors.go index a9ec3059..5d2e1730 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -1,13 +1,180 @@ package engine -import "errors" - -// IsUnsupported classifies only active unsupported engine modes already -// represented by ErrNotImplemented. -// -// It does not classify validation, domain, invariant, runtime, catalog, or -// storage failures. It also does not imply deferred candidate-only surfaces are -// active. The helper remains intentionally narrow. +import ( + "context" + "errors" + + "github.com/franchoy/coldkeep/internal/invariants" +) + +// ErrorCode is the renderer- and backend-neutral classification of an engine +// operation failure. Human messages and public CLI exit behavior remain caller +// compatibility concerns. +type ErrorCode string + +const ( + ErrorInvalidArgument ErrorCode = "invalid_argument" + ErrorNotFound ErrorCode = "not_found" + ErrorUnsupported ErrorCode = "unsupported" + ErrorInvariantViolation ErrorCode = "invariant_violation" + ErrorVerificationFailed ErrorCode = "verification_failed" + ErrorRecoveryFailed ErrorCode = "recovery_failed" + ErrorConflict ErrorCode = "conflict" + ErrorCancelled ErrorCode = "cancelled" + ErrorOperationFailed ErrorCode = "operation_failed" +) + +// ErrNotImplemented is retained while the final v1.13.12 promised surfaces +// are activated. New engine operations must return a typed ErrorUnsupported +// instead of adding another untyped unsupported sentinel. +var ErrNotImplemented = errors.New("engine operation not implemented") + +// Error is the stable typed engine failure. Its exported state is deliberately +// string-only and backend-neutral; cause remains private but is available to +// errors.Is/errors.As through Unwrap. +type Error struct { + Code ErrorCode + Operation string + Message string + InvariantCode string + cause error +} + +func (e *Error) Error() string { + if e == nil { + return "" + } + if e.Message != "" { + return e.Message + } + if e.cause != nil { + return e.cause.Error() + } + return string(e.Code) +} + +func (e *Error) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +// NewError creates one typed error without changing the supplied human +// message. Invalid or empty codes fail closed to operation_failed. +func NewError(code ErrorCode, operation, message, invariantCode string, cause error) *Error { + if !validErrorCode(code) { + code = ErrorOperationFailed + } + if message == "" && cause != nil { + message = cause.Error() + } + return &Error{ + Code: code, + Operation: operation, + Message: message, + InvariantCode: invariantCode, + cause: cause, + } +} + +// TranslateError applies deterministic engine-wide classification while +// preserving the original error message and chain. Existing engine Errors are +// returned unchanged. +func TranslateError(operation string, err error) error { + if err == nil { + return nil + } + var typed *Error + if errors.As(err, &typed) { + return err + } + + code := ErrorOperationFailed + invariantCode := "" + switch { + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + code = ErrorCancelled + case errors.Is(err, ErrNotImplemented): + code = ErrorUnsupported + default: + if value, ok := invariants.Code(err); ok { + code = ErrorInvariantViolation + invariantCode = value + } + } + return NewError(code, operation, err.Error(), invariantCode, err) +} + +// TranslateErrorAs applies a caller-selected semantic classification. Context +// cancellation and invariant errors still use their universal classifications. +func TranslateErrorAs(operation string, code ErrorCode, err error) error { + if err == nil { + return nil + } + var typed *Error + if errors.As(err, &typed) { + return err + } + if universal := CodeOf(err); universal == ErrorCancelled || universal == ErrorInvariantViolation { + return TranslateError(operation, err) + } + if errors.Is(err, ErrNotImplemented) { + code = ErrorUnsupported + } + invariantCode := "" + if value, ok := invariants.Code(err); ok { + invariantCode = value + } + return NewError(code, operation, err.Error(), invariantCode, err) +} + +// CodeOf returns the typed or universally derivable classification. An empty +// result means the error has not crossed a typed engine boundary yet. +func CodeOf(err error) ErrorCode { + if err == nil { + return "" + } + var typed *Error + if errors.As(err, &typed) { + return typed.Code + } + switch { + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return ErrorCancelled + case errors.Is(err, ErrNotImplemented): + return ErrorUnsupported + } + if _, ok := invariants.Code(err); ok { + return ErrorInvariantViolation + } + return "" +} + +// IsCode reports whether err has the requested stable engine classification. +func IsCode(err error, code ErrorCode) bool { + return code != "" && CodeOf(err) == code +} + +// IsUnsupported preserves the original sentinel classifier while recognizing +// the typed replacement used by newly activated operations. func IsUnsupported(err error) bool { - return errors.Is(err, ErrNotImplemented) + return errors.Is(err, ErrNotImplemented) || IsCode(err, ErrorUnsupported) +} + +func validErrorCode(code ErrorCode) bool { + switch code { + case ErrorInvalidArgument, + ErrorNotFound, + ErrorUnsupported, + ErrorInvariantViolation, + ErrorVerificationFailed, + ErrorRecoveryFailed, + ErrorConflict, + ErrorCancelled, + ErrorOperationFailed: + return true + default: + return false + } } diff --git a/internal/engine/errors_test.go b/internal/engine/errors_test.go index 8809a4f6..a6774423 100644 --- a/internal/engine/errors_test.go +++ b/internal/engine/errors_test.go @@ -8,6 +8,7 @@ import ( "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/storage" ) @@ -66,3 +67,104 @@ func TestUnsupportedEngineModesRemainRecognizedByIsUnsupported(t *testing.T) { } }) } + +func TestEngineErrorCodesAreStable(t *testing.T) { + want := map[engine.ErrorCode]string{ + engine.ErrorInvalidArgument: "invalid_argument", + engine.ErrorNotFound: "not_found", + engine.ErrorUnsupported: "unsupported", + engine.ErrorInvariantViolation: "invariant_violation", + engine.ErrorVerificationFailed: "verification_failed", + engine.ErrorRecoveryFailed: "recovery_failed", + engine.ErrorConflict: "conflict", + engine.ErrorCancelled: "cancelled", + engine.ErrorOperationFailed: "operation_failed", + } + for code, text := range want { + if string(code) != text { + t.Errorf("error code changed: got=%q want=%q", code, text) + } + } +} + +func TestNewErrorPreservesMessageFieldsAndCause(t *testing.T) { + cause := errors.New("storage unavailable") + err := engine.NewError(engine.ErrorConflict, "store", "repository busy", "LOCK_BUSY", cause) + if err.Error() != "repository busy" { + t.Fatalf("message changed: %q", err.Error()) + } + if err.Code != engine.ErrorConflict || err.Operation != "store" || err.InvariantCode != "LOCK_BUSY" { + t.Fatalf("unexpected typed fields: %+v", err) + } + if !errors.Is(err, cause) { + t.Fatal("typed error must preserve errors.Is cause chain") + } + if !engine.IsCode(err, engine.ErrorConflict) || engine.CodeOf(err) != engine.ErrorConflict { + t.Fatalf("typed classification failed: %v", err) + } +} + +func TestNewErrorRejectsUnknownCodeFailClosed(t *testing.T) { + err := engine.NewError(engine.ErrorCode("invented"), "inspect", "boom", "", nil) + if err.Code != engine.ErrorOperationFailed { + t.Fatalf("unknown code should fail closed: %+v", err) + } +} + +func TestTranslateErrorClassifiesUniversalFailures(t *testing.T) { + invariant := invariants.New(invariants.CodeGCRefusedIntegrity, "GC refused", nil) + tests := []struct { + name string + err error + want engine.ErrorCode + invariantCode string + }{ + {name: "cancelled", err: context.Canceled, want: engine.ErrorCancelled}, + {name: "deadline", err: context.DeadlineExceeded, want: engine.ErrorCancelled}, + {name: "unsupported", err: fmt.Errorf("recursive: %w", engine.ErrNotImplemented), want: engine.ErrorUnsupported}, + {name: "invariant", err: invariant, want: engine.ErrorInvariantViolation, invariantCode: invariants.CodeGCRefusedIntegrity}, + {name: "ordinary", err: errors.New("disk unavailable"), want: engine.ErrorOperationFailed}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + translated := engine.TranslateError("operation", tc.err) + if !engine.IsCode(translated, tc.want) { + t.Fatalf("code mismatch: got=%q want=%q err=%v", engine.CodeOf(translated), tc.want, translated) + } + if translated.Error() != tc.err.Error() || !errors.Is(translated, tc.err) { + t.Fatalf("translation changed message or chain: translated=%v original=%v", translated, tc.err) + } + var typed *engine.Error + if !errors.As(translated, &typed) || typed.InvariantCode != tc.invariantCode { + t.Fatalf("typed detail mismatch: %+v", typed) + } + }) + } +} + +func TestTranslateErrorAsPreservesUniversalAndExplicitClassifications(t *testing.T) { + notFound := errors.New("file not found") + translated := engine.TranslateErrorAs("restore", engine.ErrorNotFound, notFound) + if !engine.IsCode(translated, engine.ErrorNotFound) || translated.Error() != notFound.Error() { + t.Fatalf("explicit classification failed: %v", translated) + } + + cancelled := engine.TranslateErrorAs("restore", engine.ErrorNotFound, context.Canceled) + if !engine.IsCode(cancelled, engine.ErrorCancelled) { + t.Fatalf("cancellation must retain universal classification: %v", cancelled) + } + + existing := engine.NewError(engine.ErrorConflict, "gc", "busy", "", nil) + if got := engine.TranslateErrorAs("restore", engine.ErrorNotFound, existing); got != existing { + t.Fatal("existing typed error must not be reclassified") + } +} + +func TestTranslateErrorNilIsNil(t *testing.T) { + if got := engine.TranslateError("store", nil); got != nil { + t.Fatalf("nil translation = %v", got) + } + if got := engine.TranslateErrorAs("store", engine.ErrorInvalidArgument, nil); got != nil { + t.Fatalf("nil explicit translation = %v", got) + } +} From 2d54d582d4cd207554d917222da70de87a637f5d Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 12:34:13 +0200 Subject: [PATCH 04/27] engine: neutralize stats inspect trace and verify contracts --- CHANGELOG.md | 4 + cmd/coldkeep/main.go | 7 +- cmd/coldkeep/observability_engine_adapters.go | 188 ++++++++++++ .../observability_engine_adapters_test.go | 42 +++ docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...nspect-trace-and-verification-contracts.md | 49 ++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 12 +- .../engine/contract_surface_clarity_test.go | 11 +- internal/engine/contracts_test.go | 24 +- internal/engine/default_engine.go | 49 +++- internal/engine/engine_test.go | 35 ++- internal/engine/observability_adapter.go | 267 ++++++++++++++++++ internal/engine/observability_adapter_test.go | 76 +++++ .../engine/read_side_backend_contract_test.go | 56 ++-- internal/engine/types.go | 248 ++++++++++++++-- internal/engine/verify_summary.go | 96 +++++++ 18 files changed, 1061 insertions(+), 117 deletions(-) create mode 100644 cmd/coldkeep/observability_engine_adapters.go create mode 100644 cmd/coldkeep/observability_engine_adapters_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase3-stats-inspect-trace-and-verification-contracts.md create mode 100644 internal/engine/observability_adapter.go create mode 100644 internal/engine/observability_adapter_test.go create mode 100644 internal/engine/verify_summary.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 321eb3dd..df9fdfe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ project, do not start here; start with [README.md](README.md). translation helpers, invariant/cancellation preservation, and exhaustive active-contract neutrality coverage with only the four explicit Phase 3 observability DTO debts temporarily allowlisted. +- Replaced Stats and Inspect observability-backed contracts with complete + engine-owned DTOs, exact tagged dynamic values, ordered trace events, and a + complete Verify summary while preserving the existing renderer output + through compatibility projection. ------------------------------------------------------------------------ diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 467829e3..86b3d39b 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -504,13 +504,16 @@ var runObservabilityStatsPhase = func(opts observability.StatsOptions) (*observa result, err := eng.Stats(context.Background(), engine.StatsRequest{ IncludeContainers: opts.IncludeContainers, - Trace: opts.Trace, + IncludeTrace: opts.Trace.Enabled, }) + if traceErr := replayEngineTrace(opts.Trace, result.Trace); traceErr != nil { + return nil, traceErr + } if err != nil { return nil, err } - return result.Raw, nil + return statsResultFromEngine(result), nil } var runObservabilityInspectPhase = func(entity observability.EntityType, id string, opts observability.InspectOptions) (*observability.InspectResult, error) { // Engine.Inspect is active, while this production CLI path intentionally diff --git a/cmd/coldkeep/observability_engine_adapters.go b/cmd/coldkeep/observability_engine_adapters.go new file mode 100644 index 00000000..c51b7699 --- /dev/null +++ b/cmd/coldkeep/observability_engine_adapters.go @@ -0,0 +1,188 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/observability" +) + +func statsResultFromEngine(input engine.StatsResult) *observability.StatsResult { + versions := make([]observability.VersionStat, len(input.Chunks.ChunkerVersions)) + for i, item := range input.Chunks.ChunkerVersions { + versions[i] = observability.VersionStat{Version: item.Version, Chunks: item.Chunks, Bytes: item.Bytes} + } + records := make([]observability.ContainerStatRecord, len(input.Containers.Records)) + for i, item := range input.Containers.Records { + records[i] = observability.ContainerStatRecord{ + ID: item.ID, Filename: item.Filename, TotalBytes: item.TotalBytes, + LiveBytes: item.LiveBytes, DeadBytes: item.DeadBytes, + Quarantine: item.Quarantine, LiveRatioPct: item.LiveRatioPct, + } + } + warnings := make([]observability.ObservationWarning, len(input.Warnings)) + for i, item := range input.Warnings { + warnings[i] = observability.ObservationWarning{Code: item.Code, Message: item.Message} + } + return &observability.StatsResult{ + GeneratedAtUTC: input.GeneratedAtUTC, + Repository: observability.RepositoryStats{ActiveWriteChunker: input.Repository.ActiveWriteChunker}, + Logical: observability.LogicalStats{ + TotalFiles: input.Logical.TotalFiles, CompletedFiles: input.Logical.CompletedFiles, + ProcessingFiles: input.Logical.ProcessingFiles, AbortedFiles: input.Logical.AbortedFiles, + TotalSizeBytes: input.Logical.TotalSizeBytes, CompletedSizeBytes: input.Logical.CompletedSizeBytes, + EstimatedDedupRatioPct: input.Logical.EstimatedDedupRatioPct, + }, + Physical: observability.PhysicalStats{TotalPhysicalFiles: input.Physical.TotalPhysicalFiles}, + Chunks: observability.ChunkStats{ + TotalChunks: input.Chunks.TotalChunks, CompletedChunks: input.Chunks.CompletedChunks, + CompletedBytes: input.Chunks.CompletedBytes, + CountsByVersion: cloneStringInt64ForCLI(input.Chunks.CountsByVersion), + BytesByVersion: cloneStringInt64ForCLI(input.Chunks.BytesByVersion), + ChunkerVersions: versions, TotalReferences: input.Chunks.TotalReferences, + UniqueReferenced: input.Chunks.UniqueReferenced, + }, + BlockLayout: observability.BlockLayoutStats{ + StorageBlocksCount: input.BlockLayout.StorageBlocksCount, + ChunkBlockRefsCount: input.BlockLayout.ChunkBlockRefsCount, + AvgChunksPerBlock: input.BlockLayout.AvgChunksPerBlock, + AvgBlockPlaintextSize: input.BlockLayout.AvgBlockPlaintextSize, + AvgBlockStoredSize: input.BlockLayout.AvgBlockStoredSize, + LogicalBytes: input.BlockLayout.LogicalBytes, + CompressedBytes: input.BlockLayout.CompressedBytes, + StoredBytes: input.BlockLayout.StoredBytes, + CompressionSizeRatio: input.BlockLayout.CompressionSizeRatio, + CompressionFactor: input.BlockLayout.CompressionFactor, + PhysicalSizeRatio: input.BlockLayout.PhysicalSizeRatio, + PhysicalFactor: input.BlockLayout.PhysicalFactor, + CompressedBlocks: input.BlockLayout.CompressedBlocks, + UncompressedBlocks: input.BlockLayout.UncompressedBlocks, + CompressionCodecBreakdown: cloneStringInt64ForCLI(input.BlockLayout.CompressionCodecBreakdown), + AvgBlockFillRatio: input.BlockLayout.AvgBlockFillRatio, + LegacyBlockCount: input.BlockLayout.LegacyBlockCount, + PackedBlockCount: input.BlockLayout.PackedBlockCount, + CodecDistribution: cloneStringInt64ForCLI(input.BlockLayout.CodecDistribution), + }, + Containers: observability.ContainerStats{ + TotalContainers: input.Containers.TotalContainers, + HealthyContainers: input.Containers.HealthyContainers, + QuarantineContainers: input.Containers.QuarantineContainers, + TotalBytes: input.Containers.TotalBytes, HealthyBytes: input.Containers.HealthyBytes, + QuarantineBytes: input.Containers.QuarantineBytes, + LiveBlockBytes: input.Containers.LiveBlockBytes, DeadBlockBytes: input.Containers.DeadBlockBytes, + FragmentationRatioPct: input.Containers.FragmentationRatioPct, Records: records, + }, + Efficiency: observability.EfficiencyStats{ + LogicalBytes: input.Efficiency.LogicalBytes, + UniqueChunkBytes: input.Efficiency.UniqueChunkBytes, + ContainerBytes: input.Efficiency.ContainerBytes, + DedupRatio: input.Efficiency.DedupRatio, + DedupRatioPercent: input.Efficiency.DedupRatioPercent, + ContainerOverheadPct: input.Efficiency.ContainerOverheadPct, + StorageOverheadPct: input.Efficiency.StorageOverheadPct, + }, + Snapshots: observability.SnapshotStats{TotalSnapshots: input.Snapshots.TotalSnapshots}, + Retention: observability.RetentionStats{ + CurrentOnlyLogicalFiles: input.Retention.CurrentOnlyLogicalFiles, + CurrentOnlyBytes: input.Retention.CurrentOnlyBytes, + SnapshotReferencedLogicalFiles: input.Retention.SnapshotReferencedLogicalFiles, + SnapshotReferencedBytes: input.Retention.SnapshotReferencedBytes, + SnapshotOnlyLogicalFiles: input.Retention.SnapshotOnlyLogicalFiles, + SnapshotOnlyBytes: input.Retention.SnapshotOnlyBytes, + SharedLogicalFiles: input.Retention.SharedLogicalFiles, + SharedBytes: input.Retention.SharedBytes, + }, + Graph: observability.GraphStats{ + SnapshotReachableChunks: input.Graph.SnapshotReachableChunks, + SnapshotReachableBytes: input.Graph.SnapshotReachableBytes, + }, + Warnings: warnings, + } +} + +func replayEngineTrace(options observability.TraceOptions, events []engine.TraceEvent) error { + if !options.Enabled || options.Sink == nil { + return nil + } + for _, item := range events { + metadata, err := engineValuesToAny(item.Metadata) + if err != nil { + return fmt.Errorf("replay trace event %q: %w", item.Step, err) + } + options.Sink.Event(observability.TraceEvent{ + Step: item.Step, Entity: item.Entity, EntityID: item.EntityID, + Message: item.Message, Metadata: metadata, + }) + } + return nil +} + +func engineValuesToAny(input map[string]engine.Value) (map[string]any, error) { + if len(input) == 0 { + return nil, nil + } + out := make(map[string]any, len(input)) + for key, item := range input { + value, err := engineValueToAny(item) + if err != nil { + return nil, fmt.Errorf("field %q: %w", key, err) + } + out[key] = value + } + return out, nil +} + +func engineValueToAny(input engine.Value) (any, error) { + switch input.Kind { + case engine.ValueNull: + return nil, nil + case engine.ValueBoolean: + return input.Boolean, nil + case engine.ValueString: + return input.String, nil + case engine.ValueInteger: + if value, err := strconv.ParseInt(input.Integer, 10, 64); err == nil { + return value, nil + } + if value, err := strconv.ParseUint(input.Integer, 10, 64); err == nil { + return value, nil + } + if _, err := json.Number(input.Integer).Float64(); err != nil { + return nil, fmt.Errorf("invalid integer %q", input.Integer) + } + return json.Number(input.Integer), nil + case engine.ValueDecimal: + value, err := strconv.ParseFloat(input.Decimal, 64) + if err != nil { + return nil, fmt.Errorf("invalid decimal %q", input.Decimal) + } + return value, nil + case engine.ValueObject: + return engineValuesToAny(input.Object) + case engine.ValueArray: + items := make([]any, len(input.Array)) + for i, item := range input.Array { + value, err := engineValueToAny(item) + if err != nil { + return nil, fmt.Errorf("array item %d: %w", i, err) + } + items[i] = value + } + return items, nil + default: + return nil, fmt.Errorf("unknown engine value kind %q", input.Kind) + } +} + +func cloneStringInt64ForCLI(input map[string]int64) map[string]int64 { + if input == nil { + return nil + } + out := make(map[string]int64, len(input)) + for key, value := range input { + out[key] = value + } + return out +} diff --git a/cmd/coldkeep/observability_engine_adapters_test.go b/cmd/coldkeep/observability_engine_adapters_test.go new file mode 100644 index 00000000..7b6293aa --- /dev/null +++ b/cmd/coldkeep/observability_engine_adapters_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" +) + +func TestEngineValueToAnyPreservesLargeIntegerJSONToken(t *testing.T) { + value, err := engineValueToAny(engine.Value{Kind: engine.ValueInteger, Integer: "18446744073709551615"}) + if err != nil { + t.Fatalf("engineValueToAny: %v", err) + } + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + if got, want := string(encoded), "18446744073709551615"; got != want { + t.Fatalf("integer token changed: got=%s want=%s", got, want) + } +} + +func TestStatsEngineCompatibilityProjectionPreservesFields(t *testing.T) { + projected := statsResultFromEngine(engine.StatsResult{ + Logical: engine.StatsLogical{TotalFiles: 3, TotalSizeBytes: 99}, + Chunks: engine.StatsChunks{ + CountsByVersion: map[string]int64{"v2-fastcdc": 2}, + ChunkerVersions: []engine.StatsVersion{{Version: "v2-fastcdc", Chunks: 2, Bytes: 99}}, + }, + Warnings: []engine.OperationWarning{{Code: "warning", Message: "detail"}}, + }) + if projected.Logical.TotalFiles != 3 || projected.Logical.TotalSizeBytes != 99 { + t.Fatalf("logical projection changed: %+v", projected.Logical) + } + if projected.Chunks.CountsByVersion["v2-fastcdc"] != 2 || len(projected.Chunks.ChunkerVersions) != 1 { + t.Fatalf("chunk projection changed: %+v", projected.Chunks) + } + if len(projected.Warnings) != 1 || projected.Warnings[0].Code != "warning" { + t.Fatalf("warning projection changed: %+v", projected.Warnings) + } +} diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index d077a5aa..9ad94870 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–2 are complete; Phase 3 is next. + 0–3 are complete; Phase 4 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 129f97e6..ed2aa6d0 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -27,11 +27,13 @@ ## Phase 3 — Stats, Inspect, Trace, and Verification Contracts -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 3 neutral read contracts](v1.13.12-phase3-stats-inspect-trace-and-verification-contracts.md). ## Phase 4 — Catalog Contract Replacement -**Status:** Not started +**Status:** Next ## Phase 5 — Snapshot Graph Implementation diff --git a/docs/release/v1.13/v1.13.12-phase3-stats-inspect-trace-and-verification-contracts.md b/docs/release/v1.13/v1.13.12-phase3-stats-inspect-trace-and-verification-contracts.md new file mode 100644 index 00000000..4e2b7939 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase3-stats-inspect-trace-and-verification-contracts.md @@ -0,0 +1,49 @@ +# Coldkeep v1.13.12 Phase 3 — Stats, Inspect, Trace, and Verification Contracts + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `3 — Stats, Inspect, Trace, and Verification Contracts` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Neutral contracts + +Stats now returns complete engine-owned repository, logical, physical, chunk, +block-layout, container, efficiency, snapshot, retention, graph, warning, and +trace DTOs. Inspect now uses engine-owned entity/options/relation/result types +and `map[string]Value` instead of `map[string]any`. + +`Value` is a recursive tagged representation for null, Boolean, string, +integer, decimal, object, and array values. Integer and decimal values retain +their exact textual token, including values above JavaScript's exact integer +range. Non-finite and unsupported values fail closed during conversion. + +Requests use `IncludeTrace` rather than accepting a sink interface. The engine +collects already-sanitized observability events in order and returns neutral +trace events. The CLI compatibility adapter replays those events through the +existing human or JSON sink, retaining current output without putting a writer +or renderer in an engine contract. + +Verify now returns the five existing operator counters after successful +verification: blocks, physical hashes, compressed hashes, logical hashes, and +compressed blocks checked. The CLI keeps its historical summary query until +the routing-only cleanup in Phase 14, so this phase does not change output. + +## Compatibility + +Stats is already engine-routed; a lossless adapter projects its new neutral +result into the existing renderer input. Inspect remains on its existing CLI +route until Phase 14. No command syntax, JSON field, integer token, trace line, +exit code, schema, storage byte, transaction, or backend default changed. + +## Validation + +- `go test -count=1 ./internal/engine`: pass. +- `go test -count=1 ./cmd/coldkeep`: pass. +- `go test -count=1 ./internal/cli/render ./internal/observability`: pass. +- Exact numeric, nested object/array, non-finite rejection, trace ordering, and + compatibility projection tests: pass. +- Validation state: `/tmp/coldkeep-v11312-phase3.5VDbOM/` with separate + package storage roots and Go caches. + +Phase 4 is authorized to replace catalog contract shapes only. It may not yet +adopt the new plans in restore or GC production behavior. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index df657895..c56dbaa1 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–2 are Complete. Phase 3 is Next: replace observability-backed stats, -inspect, trace, and verification shapes with engine-neutral DTOs before any -CLI routing changes. +Phases 0–3 are Complete. Phase 4 is Next: replace the provisional snapshot +graph, placement, restore-plan, and GC-plan catalog shapes with complete +neutral contracts before implementing their queries. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 0e2bb4d5..4330b480 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -47,11 +47,19 @@ ## Phase 3 — Stats, Inspect, Trace, and Verification Contracts -**Phase status:** Next +**Phase status:** Complete + +- [x] Stats request/result contains no observability or renderer type. +- [x] Inspect request/result contains no observability type or `any` field. +- [x] Dynamic integer/decimal values preserve exact tokens. +- [x] Trace sinks are replaced by ordered neutral result events. +- [x] Verify returns all existing CLI summary counters. +- [x] Compatibility projection preserves current human/JSON/trace output. +- [x] The Phase 2 neutrality exceptions are removed. ## Phase 4 — Catalog Contract Replacement -**Phase status:** Not started +**Phase status:** Next ## Phase 5 — Snapshot Graph Implementation diff --git a/internal/engine/contract_surface_clarity_test.go b/internal/engine/contract_surface_clarity_test.go index 9c5cd8c0..3c0b4275 100644 --- a/internal/engine/contract_surface_clarity_test.go +++ b/internal/engine/contract_surface_clarity_test.go @@ -86,9 +86,12 @@ func TestBuildSnapshotDiffResultDetailedModeKeepsMatchedAndTotalDistinct(t *test } } -func TestVerifyResultRemainsIntentionalPlaceholder(t *testing.T) { - var result VerifyResult - if result != (VerifyResult{}) { - t.Fatalf("expected empty VerifyResult placeholder, got %#v", result) +func TestVerifyResultRepresentsCompleteCLISummary(t *testing.T) { + result := VerifyResult{ + BlocksChecked: 5, PhysicalHashChecked: 4, CompressedHashChecked: 3, + LogicalHashChecked: 2, CompressedBlocksChecked: 1, + } + if result.BlocksChecked != 5 || result.CompressedBlocksChecked != 1 { + t.Fatalf("verify summary not representable: %#v", result) } } diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 10c8ddbd..361a58bd 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -34,6 +34,10 @@ func activeCoreContractTypes() []struct { }{ {"OperationWarning", engine.OperationWarning{}}, {"Error", engine.Error{}}, + {"StatsRequest", engine.StatsRequest{}}, + {"StatsResult", engine.StatsResult{}}, + {"InspectRequest", engine.InspectRequest{}}, + {"InspectResult", engine.InspectResult{}}, {"BatchSummary", engine.BatchSummary{}}, {"SnapshotQuery", engine.SnapshotQuery{}}, {"StoreRequest", engine.StoreRequest{}}, @@ -69,23 +73,14 @@ func activeCoreContractTypes() []struct { } // TestActiveEngineContractNeutralityCoverage prevents a request/result pair -// from escaping the structural neutrality walk when a method is added. The -// four exact observability-backed DTOs are the frozen Phase 3 migration debt; -// no other omission is permitted, and Phase 3 must remove this allowlist. +// from escaping the structural neutrality walk when a method is added. func TestActiveEngineContractNeutralityCoverage(t *testing.T) { covered := make(map[reflect.Type]bool) for _, tc := range allEngineContractTypes() { covered[reflect.TypeOf(tc.val)] = true } - legacyPhase3 := map[reflect.Type]bool{ - reflect.TypeOf(engine.StatsRequest{}): true, - reflect.TypeOf(engine.StatsResult{}): true, - reflect.TypeOf(engine.InspectRequest{}): true, - reflect.TypeOf(engine.InspectResult{}): true, - } interfaceType := reflect.TypeOf((*engine.Engine)(nil)).Elem() - seenLegacy := make(map[reflect.Type]bool) for i := 0; i < interfaceType.NumMethod(); i++ { method := interfaceType.Method(i) if method.Type.NumIn() != 2 || method.Type.NumOut() != 2 { @@ -95,18 +90,9 @@ func TestActiveEngineContractNeutralityCoverage(t *testing.T) { if covered[contractType] { continue } - if legacyPhase3[contractType] { - seenLegacy[contractType] = true - continue - } t.Errorf("Engine.%s contract %s is absent from the neutrality walk", method.Name, contractType) } } - for contractType := range legacyPhase3 { - if !seenLegacy[contractType] { - t.Errorf("stale Phase 3 neutrality exception for %s", contractType) - } - } } func activeSnapshotMutationContractTypes() []struct { diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index bcf6fc46..5880e535 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "database/sql" "encoding/hex" + "errors" "fmt" "regexp" "strconv" @@ -76,25 +77,44 @@ func secureSnapshotIDGenerator() (string, error) { } func (e *DefaultEngine) Stats(ctx context.Context, req StatsRequest) (StatsResult, error) { + trace, collector := traceOptions(req.IncludeTrace) r, err := e.obs.Stats(ctx, observability.StatsOptions{ IncludeContainers: req.IncludeContainers, - Trace: req.Trace, + Trace: trace, }) + result := statsFromObservability(r, collector.events) + if collector.err != nil { + return result, TranslateError("stats", collector.err) + } if err != nil { - return StatsResult{}, err + return result, TranslateError("stats", err) } - return StatsResult{Raw: r}, nil + return result, nil } func (e *DefaultEngine) Inspect(ctx context.Context, req InspectRequest) (InspectResult, error) { if err := validateInspectRequest(req); err != nil { - return InspectResult{}, err + return InspectResult{}, TranslateErrorAs("inspect", ErrorInvalidArgument, err) + } + trace, collector := traceOptions(req.Options.IncludeTrace) + r, err := e.obs.Inspect(ctx, observability.EntityType(req.Entity), req.EntityID, observability.InspectOptions{ + Deep: req.Options.Deep, Relations: req.Options.Relations, + Reverse: req.Options.Reverse, Limit: req.Options.Limit, Trace: trace, + }) + result, conversionErr := inspectFromObservability(r, collector.events) + if collector.err != nil { + return result, TranslateError("inspect", collector.err) + } + if conversionErr != nil { + return result, TranslateError("inspect", conversionErr) } - r, err := e.obs.Inspect(ctx, req.Entity, req.EntityID, req.Options) if err != nil { - return InspectResult{}, err + if errors.Is(err, observability.ErrNotFound) || errors.Is(err, sql.ErrNoRows) { + return result, TranslateErrorAs("inspect", ErrorNotFound, err) + } + return result, TranslateError("inspect", err) } - return InspectResult{Raw: r}, nil + return result, nil } func (e *DefaultEngine) Verify(ctx context.Context, req VerifyRequest) (VerifyResult, error) { @@ -117,9 +137,13 @@ func (e *DefaultEngine) Verify(ctx context.Context, req VerifyRequest) (VerifyRe containerDir = container.ContainersDir } if err := maintenance.VerifyCommandWithDBAndContainersDir(e.config.DB, containerDir, target, req.FileID, level); err != nil { - return VerifyResult{}, err + return VerifyResult{}, TranslateErrorAs("verify", ErrorVerificationFailed, err) } - return VerifyResult{}, nil + result, err := collectVerifyResult(ctx, e.config.DB, target, int64(req.FileID)) + if err != nil { + return VerifyResult{}, TranslateError("verify", fmt.Errorf("collect verify summary: %w", err)) + } + return result, nil } // validateInspectRequest returns an error if req contains an unrecognized entity @@ -127,16 +151,15 @@ func (e *DefaultEngine) Verify(ctx context.Context, req VerifyRequest) (VerifyRe // validation so correctness does not depend solely on the CLI parsing path. func validateInspectRequest(req InspectRequest) error { switch req.Entity { - case observability.EntityRepository: + case InspectRepository: // EntityRepository is the only entity that requires no ID. return nil - case observability.EntitySnapshot: + case InspectSnapshot: if strings.TrimSpace(req.EntityID) == "" { return fmt.Errorf("engine: entity ID is required for %s", req.Entity) } return nil - case observability.EntityFile, observability.EntityLogicalFile, observability.EntityPhysicalFile, - observability.EntityChunk, observability.EntityContainer: + case InspectFile, InspectLogicalFile, InspectPhysicalFile, InspectChunk, InspectContainer: id := strings.TrimSpace(req.EntityID) if id == "" { return fmt.Errorf("engine: entity ID is required for %s", req.Entity) diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 702be2b8..c2bfd8c7 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -5,7 +5,6 @@ import ( "reflect" "testing" - "github.com/franchoy/coldkeep/internal/observability" "github.com/franchoy/coldkeep/internal/verify" ) @@ -130,23 +129,23 @@ func TestValidateInspectRequest(t *testing.T) { req InspectRequest wantErr bool }{ - {"repository no id", InspectRequest{Entity: observability.EntityRepository}, false}, - {"repository id ignored", InspectRequest{Entity: observability.EntityRepository, EntityID: "99"}, false}, - {"file positive int", InspectRequest{Entity: observability.EntityFile, EntityID: "42"}, false}, - {"file zero", InspectRequest{Entity: observability.EntityFile, EntityID: "0"}, true}, - {"file negative", InspectRequest{Entity: observability.EntityFile, EntityID: "-1"}, true}, - {"file non-numeric", InspectRequest{Entity: observability.EntityFile, EntityID: "abc"}, true}, - {"file empty id", InspectRequest{Entity: observability.EntityFile, EntityID: ""}, true}, - {"file whitespace id", InspectRequest{Entity: observability.EntityFile, EntityID: " "}, true}, - {"logical_file positive int", InspectRequest{Entity: observability.EntityLogicalFile, EntityID: "1"}, false}, - {"physical_file positive int", InspectRequest{Entity: observability.EntityPhysicalFile, EntityID: "1"}, false}, - {"chunk positive int", InspectRequest{Entity: observability.EntityChunk, EntityID: "7"}, false}, - {"container positive int", InspectRequest{Entity: observability.EntityContainer, EntityID: "3"}, false}, - {"snapshot string id", InspectRequest{Entity: observability.EntitySnapshot, EntityID: "snap-123"}, false}, - {"snapshot empty id", InspectRequest{Entity: observability.EntitySnapshot, EntityID: ""}, true}, - {"snapshot whitespace id", InspectRequest{Entity: observability.EntitySnapshot, EntityID: " "}, true}, - {"unknown entity", InspectRequest{Entity: observability.EntityType("bogus")}, true}, - {"empty entity", InspectRequest{Entity: observability.EntityType("")}, true}, + {"repository no id", InspectRequest{Entity: InspectRepository}, false}, + {"repository id ignored", InspectRequest{Entity: InspectRepository, EntityID: "99"}, false}, + {"file positive int", InspectRequest{Entity: InspectFile, EntityID: "42"}, false}, + {"file zero", InspectRequest{Entity: InspectFile, EntityID: "0"}, true}, + {"file negative", InspectRequest{Entity: InspectFile, EntityID: "-1"}, true}, + {"file non-numeric", InspectRequest{Entity: InspectFile, EntityID: "abc"}, true}, + {"file empty id", InspectRequest{Entity: InspectFile, EntityID: ""}, true}, + {"file whitespace id", InspectRequest{Entity: InspectFile, EntityID: " "}, true}, + {"logical_file positive int", InspectRequest{Entity: InspectLogicalFile, EntityID: "1"}, false}, + {"physical_file positive int", InspectRequest{Entity: InspectPhysicalFile, EntityID: "1"}, false}, + {"chunk positive int", InspectRequest{Entity: InspectChunk, EntityID: "7"}, false}, + {"container positive int", InspectRequest{Entity: InspectContainer, EntityID: "3"}, false}, + {"snapshot string id", InspectRequest{Entity: InspectSnapshot, EntityID: "snap-123"}, false}, + {"snapshot empty id", InspectRequest{Entity: InspectSnapshot, EntityID: ""}, true}, + {"snapshot whitespace id", InspectRequest{Entity: InspectSnapshot, EntityID: " "}, true}, + {"unknown entity", InspectRequest{Entity: InspectEntity("bogus")}, true}, + {"empty entity", InspectRequest{Entity: InspectEntity("")}, true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/engine/observability_adapter.go b/internal/engine/observability_adapter.go new file mode 100644 index 00000000..fc512f93 --- /dev/null +++ b/internal/engine/observability_adapter.go @@ -0,0 +1,267 @@ +package engine + +import ( + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" + + "github.com/franchoy/coldkeep/internal/observability" +) + +type observabilityTraceCollector struct { + events []TraceEvent + err error +} + +func (c *observabilityTraceCollector) Event(event observability.TraceEvent) { + if c == nil || c.err != nil { + return + } + metadata, err := valuesFromMap(event.Metadata) + if err != nil { + c.err = fmt.Errorf("convert trace event %q metadata: %w", event.Step, err) + return + } + c.events = append(c.events, TraceEvent{ + Step: event.Step, + Entity: event.Entity, + EntityID: event.EntityID, + Message: event.Message, + Metadata: metadata, + }) +} + +func traceOptions(enabled bool) (observability.TraceOptions, *observabilityTraceCollector) { + collector := &observabilityTraceCollector{} + if !enabled { + return observability.TraceOptions{}, collector + } + return observability.TraceOptions{Enabled: true, Sink: collector}, collector +} + +func statsFromObservability(raw *observability.StatsResult, trace []TraceEvent) StatsResult { + if raw == nil { + return StatsResult{Trace: cloneTraceEvents(trace)} + } + versions := make([]StatsVersion, len(raw.Chunks.ChunkerVersions)) + for i, item := range raw.Chunks.ChunkerVersions { + versions[i] = StatsVersion{Version: item.Version, Chunks: item.Chunks, Bytes: item.Bytes} + } + records := make([]StatsContainerRecord, len(raw.Containers.Records)) + for i, item := range raw.Containers.Records { + records[i] = StatsContainerRecord{ + ID: item.ID, Filename: item.Filename, TotalBytes: item.TotalBytes, + LiveBytes: item.LiveBytes, DeadBytes: item.DeadBytes, + Quarantine: item.Quarantine, LiveRatioPct: item.LiveRatioPct, + } + } + warnings := make([]OperationWarning, len(raw.Warnings)) + for i, item := range raw.Warnings { + warnings[i] = OperationWarning{Code: item.Code, Message: item.Message} + } + return StatsResult{ + GeneratedAtUTC: raw.GeneratedAtUTC, + Repository: StatsRepository{ActiveWriteChunker: raw.Repository.ActiveWriteChunker}, + Logical: StatsLogical{ + TotalFiles: raw.Logical.TotalFiles, CompletedFiles: raw.Logical.CompletedFiles, + ProcessingFiles: raw.Logical.ProcessingFiles, AbortedFiles: raw.Logical.AbortedFiles, + TotalSizeBytes: raw.Logical.TotalSizeBytes, CompletedSizeBytes: raw.Logical.CompletedSizeBytes, + EstimatedDedupRatioPct: raw.Logical.EstimatedDedupRatioPct, + }, + Physical: StatsPhysical{TotalPhysicalFiles: raw.Physical.TotalPhysicalFiles}, + Chunks: StatsChunks{ + TotalChunks: raw.Chunks.TotalChunks, CompletedChunks: raw.Chunks.CompletedChunks, + CompletedBytes: raw.Chunks.CompletedBytes, + CountsByVersion: cloneStringInt64Map(raw.Chunks.CountsByVersion), + BytesByVersion: cloneStringInt64Map(raw.Chunks.BytesByVersion), + ChunkerVersions: versions, TotalReferences: raw.Chunks.TotalReferences, + UniqueReferenced: raw.Chunks.UniqueReferenced, + }, + BlockLayout: StatsBlockLayout{ + StorageBlocksCount: raw.BlockLayout.StorageBlocksCount, + ChunkBlockRefsCount: raw.BlockLayout.ChunkBlockRefsCount, + AvgChunksPerBlock: raw.BlockLayout.AvgChunksPerBlock, + AvgBlockPlaintextSize: raw.BlockLayout.AvgBlockPlaintextSize, + AvgBlockStoredSize: raw.BlockLayout.AvgBlockStoredSize, + LogicalBytes: raw.BlockLayout.LogicalBytes, + CompressedBytes: raw.BlockLayout.CompressedBytes, + StoredBytes: raw.BlockLayout.StoredBytes, + CompressionSizeRatio: raw.BlockLayout.CompressionSizeRatio, + CompressionFactor: raw.BlockLayout.CompressionFactor, + PhysicalSizeRatio: raw.BlockLayout.PhysicalSizeRatio, + PhysicalFactor: raw.BlockLayout.PhysicalFactor, + CompressedBlocks: raw.BlockLayout.CompressedBlocks, + UncompressedBlocks: raw.BlockLayout.UncompressedBlocks, + CompressionCodecBreakdown: cloneStringInt64Map(raw.BlockLayout.CompressionCodecBreakdown), + AvgBlockFillRatio: raw.BlockLayout.AvgBlockFillRatio, + LegacyBlockCount: raw.BlockLayout.LegacyBlockCount, + PackedBlockCount: raw.BlockLayout.PackedBlockCount, + CodecDistribution: cloneStringInt64Map(raw.BlockLayout.CodecDistribution), + }, + Containers: StatsContainers{ + TotalContainers: raw.Containers.TotalContainers, + HealthyContainers: raw.Containers.HealthyContainers, + QuarantineContainers: raw.Containers.QuarantineContainers, + TotalBytes: raw.Containers.TotalBytes, HealthyBytes: raw.Containers.HealthyBytes, + QuarantineBytes: raw.Containers.QuarantineBytes, + LiveBlockBytes: raw.Containers.LiveBlockBytes, DeadBlockBytes: raw.Containers.DeadBlockBytes, + FragmentationRatioPct: raw.Containers.FragmentationRatioPct, Records: records, + }, + Efficiency: StatsEfficiency{ + LogicalBytes: raw.Efficiency.LogicalBytes, UniqueChunkBytes: raw.Efficiency.UniqueChunkBytes, + ContainerBytes: raw.Efficiency.ContainerBytes, DedupRatio: raw.Efficiency.DedupRatio, + DedupRatioPercent: raw.Efficiency.DedupRatioPercent, + ContainerOverheadPct: raw.Efficiency.ContainerOverheadPct, + StorageOverheadPct: raw.Efficiency.StorageOverheadPct, + }, + Snapshots: StatsSnapshots{TotalSnapshots: raw.Snapshots.TotalSnapshots}, + Retention: StatsRetention{ + CurrentOnlyLogicalFiles: raw.Retention.CurrentOnlyLogicalFiles, + CurrentOnlyBytes: raw.Retention.CurrentOnlyBytes, + SnapshotReferencedLogicalFiles: raw.Retention.SnapshotReferencedLogicalFiles, + SnapshotReferencedBytes: raw.Retention.SnapshotReferencedBytes, + SnapshotOnlyLogicalFiles: raw.Retention.SnapshotOnlyLogicalFiles, + SnapshotOnlyBytes: raw.Retention.SnapshotOnlyBytes, + SharedLogicalFiles: raw.Retention.SharedLogicalFiles, + SharedBytes: raw.Retention.SharedBytes, + }, + Graph: StatsGraph{ + SnapshotReachableChunks: raw.Graph.SnapshotReachableChunks, + SnapshotReachableBytes: raw.Graph.SnapshotReachableBytes, + }, + Warnings: warnings, + Trace: cloneTraceEvents(trace), + } +} + +func inspectFromObservability(raw *observability.InspectResult, trace []TraceEvent) (InspectResult, error) { + if raw == nil { + return InspectResult{Trace: cloneTraceEvents(trace)}, nil + } + summary, err := valuesFromMap(raw.Summary) + if err != nil { + return InspectResult{}, fmt.Errorf("convert inspect summary: %w", err) + } + metadata, err := valuesFromMap(raw.Metadata) + if err != nil { + return InspectResult{}, fmt.Errorf("convert inspect metadata: %w", err) + } + relations := make([]InspectRelation, len(raw.Relations)) + for i, item := range raw.Relations { + relationMetadata, err := valuesFromMap(item.Metadata) + if err != nil { + return InspectResult{}, fmt.Errorf("convert inspect relation %d metadata: %w", i, err) + } + relations[i] = InspectRelation{ + Type: item.Type, Direction: RelationDirection(item.Direction), + TargetType: InspectEntity(item.TargetType), TargetID: item.TargetID, + Metadata: relationMetadata, + } + } + warnings := make([]OperationWarning, len(raw.Warnings)) + for i, item := range raw.Warnings { + warnings[i] = OperationWarning{Code: item.Code, Message: item.Message} + } + return InspectResult{ + GeneratedAtUTC: raw.GeneratedAtUTC, + Entity: InspectEntity(raw.EntityType), EntityID: raw.EntityID, + Summary: summary, Metadata: metadata, Relations: relations, + Warnings: warnings, Trace: cloneTraceEvents(trace), + }, nil +} + +func valuesFromMap(input map[string]any) (map[string]Value, error) { + if len(input) == 0 { + return nil, nil + } + out := make(map[string]Value, len(input)) + for key, item := range input { + value, err := valueFromAny(item) + if err != nil { + return nil, fmt.Errorf("field %q: %w", key, err) + } + out[key] = value + } + return out, nil +} + +func valueFromAny(input any) (Value, error) { + if input == nil { + return Value{Kind: ValueNull}, nil + } + if number, ok := input.(json.Number); ok { + if _, err := number.Int64(); err == nil { + return Value{Kind: ValueInteger, Integer: string(number)}, nil + } + if _, err := number.Float64(); err != nil { + return Value{}, fmt.Errorf("invalid JSON number %q", number) + } + return Value{Kind: ValueDecimal, Decimal: string(number)}, nil + } + + value := reflect.ValueOf(input) + switch value.Kind() { + case reflect.Bool: + return Value{Kind: ValueBoolean, Boolean: value.Bool()}, nil + case reflect.String: + return Value{Kind: ValueString, String: value.String()}, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return Value{Kind: ValueInteger, Integer: strconv.FormatInt(value.Int(), 10)}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return Value{Kind: ValueInteger, Integer: strconv.FormatUint(value.Uint(), 10)}, nil + case reflect.Float32, reflect.Float64: + floating := value.Float() + if math.IsNaN(floating) || math.IsInf(floating, 0) { + return Value{}, fmt.Errorf("non-finite decimal") + } + return Value{Kind: ValueDecimal, Decimal: strconv.FormatFloat(floating, 'g', -1, value.Type().Bits())}, nil + case reflect.Map: + if value.Type().Key().Kind() != reflect.String { + return Value{}, fmt.Errorf("map key type %s is not string", value.Type().Key()) + } + object := make(map[string]Value, value.Len()) + iterator := value.MapRange() + for iterator.Next() { + item, err := valueFromAny(iterator.Value().Interface()) + if err != nil { + return Value{}, fmt.Errorf("object field %q: %w", iterator.Key().String(), err) + } + object[iterator.Key().String()] = item + } + return Value{Kind: ValueObject, Object: object}, nil + case reflect.Slice, reflect.Array: + items := make([]Value, value.Len()) + for i := 0; i < value.Len(); i++ { + item, err := valueFromAny(value.Index(i).Interface()) + if err != nil { + return Value{}, fmt.Errorf("array item %d: %w", i, err) + } + items[i] = item + } + return Value{Kind: ValueArray, Array: items}, nil + case reflect.Interface: + return valueFromAny(value.Interface()) + default: + return Value{}, fmt.Errorf("unsupported dynamic value type %T", input) + } +} + +func cloneStringInt64Map(input map[string]int64) map[string]int64 { + if input == nil { + return nil + } + out := make(map[string]int64, len(input)) + for key, value := range input { + out[key] = value + } + return out +} + +func cloneTraceEvents(input []TraceEvent) []TraceEvent { + if input == nil { + return nil + } + return append([]TraceEvent(nil), input...) +} diff --git a/internal/engine/observability_adapter_test.go b/internal/engine/observability_adapter_test.go new file mode 100644 index 00000000..8765438a --- /dev/null +++ b/internal/engine/observability_adapter_test.go @@ -0,0 +1,76 @@ +package engine + +import ( + "encoding/json" + "math" + "reflect" + "testing" + + "github.com/franchoy/coldkeep/internal/observability" +) + +func TestValueFromAnyPreservesExactNumericKinds(t *testing.T) { + tests := []struct { + name string + input any + want Value + }{ + {name: "signed", input: int64(math.MaxInt64), want: Value{Kind: ValueInteger, Integer: "9223372036854775807"}}, + {name: "unsigned", input: uint64(math.MaxUint64), want: Value{Kind: ValueInteger, Integer: "18446744073709551615"}}, + {name: "decimal", input: 1.25, want: Value{Kind: ValueDecimal, Decimal: "1.25"}}, + {name: "json integer", input: json.Number("9007199254740993"), want: Value{Kind: ValueInteger, Integer: "9007199254740993"}}, + {name: "json decimal", input: json.Number("1.125"), want: Value{Kind: ValueDecimal, Decimal: "1.125"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := valueFromAny(tc.input) + if err != nil || !reflect.DeepEqual(got, tc.want) { + t.Fatalf("valueFromAny(%T): got (%+v, %v), want %+v", tc.input, got, err, tc.want) + } + }) + } +} + +func TestValueFromAnyPreservesNestedObjectsAndArrays(t *testing.T) { + got, err := valueFromAny(map[string]any{ + "enabled": true, + "items": []any{"alpha", int64(7), nil}, + "counts": map[string]int64{"chunks": 3}, + }) + if err != nil { + t.Fatalf("valueFromAny: %v", err) + } + if got.Kind != ValueObject || got.Object["enabled"].Kind != ValueBoolean { + t.Fatalf("object conversion mismatch: %+v", got) + } + items := got.Object["items"] + if items.Kind != ValueArray || len(items.Array) != 3 || items.Array[2].Kind != ValueNull { + t.Fatalf("array conversion mismatch: %+v", items) + } + if got.Object["counts"].Object["chunks"].Integer != "3" { + t.Fatalf("nested integer mismatch: %+v", got) + } +} + +func TestValueFromAnyRejectsNonFiniteAndUnsupportedValues(t *testing.T) { + for _, input := range []any{math.NaN(), math.Inf(1), make(chan int)} { + if _, err := valueFromAny(input); err == nil { + t.Fatalf("expected %T to be rejected", input) + } + } +} + +func TestTraceCollectorPreservesOrderAndNeutralMetadata(t *testing.T) { + collector := &observabilityTraceCollector{} + collector.Event(observability.TraceEvent{Step: "first", Metadata: map[string]any{"count": int64(9007199254740993)}}) + collector.Event(observability.TraceEvent{Step: "second", Metadata: map[string]any{"ok": true}}) + if collector.err != nil { + t.Fatalf("collector error: %v", collector.err) + } + if len(collector.events) != 2 || collector.events[0].Step != "first" || collector.events[1].Step != "second" { + t.Fatalf("trace order changed: %+v", collector.events) + } + if got := collector.events[0].Metadata["count"]; got.Kind != ValueInteger || got.Integer != "9007199254740993" { + t.Fatalf("trace integer changed: %+v", got) + } +} diff --git a/internal/engine/read_side_backend_contract_test.go b/internal/engine/read_side_backend_contract_test.go index 9bead733..1c3edb9c 100644 --- a/internal/engine/read_side_backend_contract_test.go +++ b/internal/engine/read_side_backend_contract_test.go @@ -24,28 +24,28 @@ func TestEngineReadStatsAndInspectAcrossBackends(t *testing.T) { if err != nil { t.Fatalf("Stats: %v", err) } - if stats.Raw == nil || stats.Raw.Logical.TotalFiles != 2 || len(stats.Raw.Containers.Records) != 1 || stats.Raw.Snapshots.TotalSnapshots != 3 { - t.Fatalf("Stats result: %+v", stats.Raw) + if stats.Logical.TotalFiles != 2 || len(stats.Containers.Records) != 1 || stats.Snapshots.TotalSnapshots != 3 { + t.Fatalf("Stats result: %+v", stats) } - if again, err := fixture.engine.Stats(context.Background(), engine.StatsRequest{IncludeContainers: true}); err != nil || !equivalentStats(stats.Raw, again.Raw) { - t.Fatalf("repeated Stats: got (%+v, %v)", again.Raw, err) + if again, err := fixture.engine.Stats(context.Background(), engine.StatsRequest{IncludeContainers: true}); err != nil || !equivalentStats(stats, again) { + t.Fatalf("repeated Stats: got (%+v, %v)", again, err) } - assertInspectSummary(t, fixture, observability.EntityRepository, "", "total_snapshots", int64(3)) - assertInspectSummary(t, fixture, observability.EntityLogicalFile, fmt.Sprint(fixture.logicalA), "file_id", fixture.logicalA) - assertInspectSummary(t, fixture, observability.EntityChunk, fmt.Sprint(fixture.chunkA), "chunk_id", fixture.chunkA) - assertInspectSummary(t, fixture, observability.EntityContainer, fmt.Sprint(fixture.containerID), "container_id", fixture.containerID) - assertInspectSummary(t, fixture, observability.EntitySnapshot, "snap-target", "snapshot_id", "snap-target") + assertInspectSummary(t, fixture, engine.InspectRepository, "", "total_snapshots", int64(3)) + assertInspectSummary(t, fixture, engine.InspectLogicalFile, fmt.Sprint(fixture.logicalA), "file_id", fixture.logicalA) + assertInspectSummary(t, fixture, engine.InspectChunk, fmt.Sprint(fixture.chunkA), "chunk_id", fixture.chunkA) + assertInspectSummary(t, fixture, engine.InspectContainer, fmt.Sprint(fixture.containerID), "container_id", fixture.containerID) + assertInspectSummary(t, fixture, engine.InspectSnapshot, "snap-target", "snapshot_id", "snap-target") withRelations, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{ - Entity: observability.EntitySnapshot, EntityID: "snap-target", - Options: observability.InspectOptions{Relations: true, Deep: true, Limit: 10}, + Entity: engine.InspectSnapshot, EntityID: "snap-target", + Options: engine.InspectOptions{Relations: true, Deep: true, Limit: 10}, }) - if err != nil || withRelations.Raw == nil || !relationsSorted(withRelations.Raw.Relations) { - t.Fatalf("Inspect snapshot relations: got (%+v, %v)", withRelations.Raw, err) + if err != nil || !relationsSorted(withRelations.Relations) { + t.Fatalf("Inspect snapshot relations: got (%+v, %v)", withRelations, err) } - _, err = fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: observability.EntityPhysicalFile, EntityID: "1"}) + _, err = fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: engine.InspectPhysicalFile, EntityID: "1"}) if !errors.Is(err, observability.ErrUnsupportedEntity) || engine.IsUnsupported(err) || catalog.IsDeferred(err) { t.Fatalf("physical-file inspect classification: %v", err) } @@ -127,7 +127,7 @@ func TestEngineReadContextAndErrorsAcrossBackends(t *testing.T) { return err }}, {"Inspect", func(ctx context.Context) error { - _, err := fixture.engine.Inspect(ctx, engine.InspectRequest{Entity: observability.EntityRepository}) + _, err := fixture.engine.Inspect(ctx, engine.InspectRequest{Entity: engine.InspectRepository}) return err }}, {"Verify", func(ctx context.Context) error { @@ -172,25 +172,33 @@ func TestEngineReadContextAndErrorsAcrossBackends(t *testing.T) { }) } -func equivalentStats(first, second *observability.StatsResult) bool { - if first == nil || second == nil { - return first == second - } - a, b := *first, *second +func equivalentStats(first, second engine.StatsResult) bool { + a, b := first, second a.GeneratedAtUTC = time.Time{} b.GeneratedAtUTC = time.Time{} return reflect.DeepEqual(a, b) } -func assertInspectSummary(t *testing.T, fixture engineReadFixture, entity observability.EntityType, id, key string, want any) { +func assertInspectSummary(t *testing.T, fixture engineReadFixture, entity engine.InspectEntity, id, key string, want any) { t.Helper() result, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: entity, EntityID: id}) - if err != nil || result.Raw == nil || !reflect.DeepEqual(result.Raw.Summary[key], want) { - t.Fatalf("Inspect %s/%s: got (%+v, %v), want summary[%q]=%v", entity, id, result.Raw, err, key, want) + if value, ok := result.Summary[key]; err != nil || !ok || !engineValueMatches(value, want) { + t.Fatalf("Inspect %s/%s: got (%+v, %v), want summary[%q]=%v", entity, id, result, err, key, want) + } +} + +func engineValueMatches(value engine.Value, want any) bool { + switch expected := want.(type) { + case int64: + return value.Kind == engine.ValueInteger && value.Integer == fmt.Sprint(expected) + case string: + return value.Kind == engine.ValueString && value.String == expected + default: + return false } } -func relationsSorted(relations []observability.Relation) bool { +func relationsSorted(relations []engine.InspectRelation) bool { for i := 1; i < len(relations); i++ { left := fmt.Sprintf("%s|%s|%s|%s", relations[i-1].Direction, relations[i-1].Type, relations[i-1].TargetType, relations[i-1].TargetID) right := fmt.Sprintf("%s|%s|%s|%s", relations[i].Direction, relations[i].Type, relations[i].TargetType, relations[i].TargetID) diff --git a/internal/engine/types.go b/internal/engine/types.go index 9019b94b..101c28a3 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -1,53 +1,243 @@ package engine -import "github.com/franchoy/coldkeep/internal/observability" +import "time" + +// ValueKind identifies the exact neutral representation stored in Value. +type ValueKind string + +const ( + ValueNull ValueKind = "null" + ValueBoolean ValueKind = "boolean" + ValueString ValueKind = "string" + ValueInteger ValueKind = "integer" + ValueDecimal ValueKind = "decimal" + ValueObject ValueKind = "object" + ValueArray ValueKind = "array" +) + +// Value is a recursive, renderer-neutral representation for dynamic inspect +// and trace metadata. Integer and decimal text preserve exact numeric tokens +// across the engine boundary. +type Value struct { + Kind ValueKind + Boolean bool + String string + Integer string + Decimal string + Object map[string]Value + Array []Value +} + +// TraceEvent is a sanitized, ordered diagnostic event returned by an engine +// operation. Callers decide whether and how to render it. +type TraceEvent struct { + Step string + Entity string + EntityID string + Message string + Metadata map[string]Value +} // StatsRequest carries parameters for the Stats operation. type StatsRequest struct { - // IncludeContainers requests container-level statistics in the result. IncludeContainers bool - // Trace controls optional trace-event emission during stats collection. - Trace observability.TraceOptions + IncludeTrace bool } -// StatsResult carries the result of the Stats operation. +// StatsResult is the complete neutral repository-statistics result. type StatsResult struct { - // Raw is the underlying observability result. - Raw *observability.StatsResult + GeneratedAtUTC time.Time + Repository StatsRepository + Logical StatsLogical + Physical StatsPhysical + Chunks StatsChunks + BlockLayout StatsBlockLayout + Containers StatsContainers + Efficiency StatsEfficiency + Snapshots StatsSnapshots + Retention StatsRetention + Graph StatsGraph + Warnings []OperationWarning + Trace []TraceEvent +} + +type StatsRepository struct { + ActiveWriteChunker string +} + +type StatsLogical struct { + TotalFiles int64 + CompletedFiles int64 + ProcessingFiles int64 + AbortedFiles int64 + TotalSizeBytes int64 + CompletedSizeBytes int64 + EstimatedDedupRatioPct float64 +} + +type StatsPhysical struct { + TotalPhysicalFiles int64 +} + +type StatsChunks struct { + TotalChunks int64 + CompletedChunks int64 + CompletedBytes int64 + CountsByVersion map[string]int64 + BytesByVersion map[string]int64 + ChunkerVersions []StatsVersion + TotalReferences int64 + UniqueReferenced int64 +} + +type StatsBlockLayout struct { + StorageBlocksCount int64 + ChunkBlockRefsCount int64 + AvgChunksPerBlock float64 + AvgBlockPlaintextSize float64 + AvgBlockStoredSize float64 + LogicalBytes int64 + CompressedBytes int64 + StoredBytes int64 + CompressionSizeRatio float64 + CompressionFactor float64 + PhysicalSizeRatio float64 + PhysicalFactor float64 + CompressedBlocks int64 + UncompressedBlocks int64 + CompressionCodecBreakdown map[string]int64 + AvgBlockFillRatio float64 + LegacyBlockCount int64 + PackedBlockCount int64 + CodecDistribution map[string]int64 +} + +type StatsVersion struct { + Version string + Chunks int64 + Bytes int64 +} + +type StatsContainers struct { + TotalContainers int64 + HealthyContainers int64 + QuarantineContainers int64 + TotalBytes int64 + HealthyBytes int64 + QuarantineBytes int64 + LiveBlockBytes int64 + DeadBlockBytes int64 + FragmentationRatioPct float64 + Records []StatsContainerRecord +} + +type StatsContainerRecord struct { + ID int64 + Filename string + TotalBytes int64 + LiveBytes int64 + DeadBytes int64 + Quarantine bool + LiveRatioPct float64 +} + +type StatsEfficiency struct { + LogicalBytes int64 + UniqueChunkBytes int64 + ContainerBytes int64 + DedupRatio float64 + DedupRatioPercent float64 + ContainerOverheadPct float64 + StorageOverheadPct float64 +} + +type StatsSnapshots struct { + TotalSnapshots int64 +} + +type StatsRetention struct { + CurrentOnlyLogicalFiles int64 + CurrentOnlyBytes int64 + SnapshotReferencedLogicalFiles int64 + SnapshotReferencedBytes int64 + SnapshotOnlyLogicalFiles int64 + SnapshotOnlyBytes int64 + SharedLogicalFiles int64 + SharedBytes int64 +} + +type StatsGraph struct { + SnapshotReachableChunks int64 + SnapshotReachableBytes int64 +} + +// InspectEntity identifies a supported inspection target without importing +// the observability implementation contract. +type InspectEntity string + +const ( + InspectRepository InspectEntity = "repository" + InspectFile InspectEntity = "file" + InspectLogicalFile InspectEntity = "logical_file" + InspectPhysicalFile InspectEntity = "physical_file" + InspectChunk InspectEntity = "chunk" + InspectContainer InspectEntity = "container" + InspectSnapshot InspectEntity = "snapshot" +) + +type RelationDirection string + +const ( + RelationOutgoing RelationDirection = "outgoing" + RelationIncoming RelationDirection = "incoming" +) + +type InspectOptions struct { + Deep bool + Relations bool + Reverse bool + Limit int + IncludeTrace bool } -// InspectRequest carries parameters for the Inspect operation. type InspectRequest struct { - // Entity is the type of entity to inspect. - Entity observability.EntityType - // EntityID is the string identifier for the entity. - // For EntityRepository this field is ignored. + Entity InspectEntity EntityID string - // Options controls relation traversal, depth, and trace behavior. - Options observability.InspectOptions + Options InspectOptions +} + +type InspectRelation struct { + Type string + Direction RelationDirection + TargetType InspectEntity + TargetID string + Metadata map[string]Value } -// InspectResult carries the result of the Inspect operation. type InspectResult struct { - // Raw is the underlying observability result. - Raw *observability.InspectResult + GeneratedAtUTC time.Time + Entity InspectEntity + EntityID string + Summary map[string]Value + Metadata map[string]Value + Relations []InspectRelation + Warnings []OperationWarning + Trace []TraceEvent } // VerifyRequest carries parameters for the Verify operation. type VerifyRequest struct { - // Level is the verification level: "fast", "standard", "full", or "deep". - // Defaults to "standard" if empty. - Level string - // Target is the verification target: "system" or "file". - // Defaults to "system" if empty. + Level string Target string - // FileID is the logical file ID to verify when Target is "file". FileID int } -// VerifyResult is the active result type for Engine.Verify. It is intentionally -// empty and success-only: a nil error means verification passed at the -// requested level, while non-nil errors preserve the verification failure -// chain. No rich verification payload is frozen; any future result expansion -// requires an explicit contract change. -type VerifyResult struct{} +// VerifyResult includes the complete stable summary required by the existing +// CLI after successful verification. +type VerifyResult struct { + BlocksChecked int64 + PhysicalHashChecked int64 + CompressedHashChecked int64 + LogicalHashChecked int64 + CompressedBlocksChecked int64 +} diff --git a/internal/engine/verify_summary.go b/internal/engine/verify_summary.go new file mode 100644 index 00000000..25c06ed8 --- /dev/null +++ b/internal/engine/verify_summary.go @@ -0,0 +1,96 @@ +package engine + +import ( + "context" + "database/sql" + "fmt" +) + +func collectVerifyResult(ctx context.Context, dbconn *sql.DB, target string, fileID int64) (VerifyResult, error) { + if dbconn == nil { + return VerifyResult{}, fmt.Errorf("verify summary DB connection is nil") + } + switch target { + case "system": + return countVerifyResultForSystem(ctx, dbconn) + case "file": + return countVerifyResultForFile(ctx, dbconn, fileID) + default: + return VerifyResult{}, fmt.Errorf("unknown verify target: %s", target) + } +} + +func countVerifyResultForSystem(ctx context.Context, dbconn *sql.DB) (VerifyResult, error) { + var result VerifyResult + if err := dbconn.QueryRowContext(ctx, ` + SELECT + COUNT(*), + COALESCE(SUM(CASE WHEN physical_hash IS NOT NULL AND length(physical_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN compressed_hash IS NOT NULL AND length(compressed_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN block_hash IS NOT NULL AND length(block_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN lower(trim(COALESCE(compression_codec, 'none'))) != 'none' THEN 1 ELSE 0 END), 0) + FROM storage_blocks + `).Scan( + &result.BlocksChecked, + &result.PhysicalHashChecked, + &result.CompressedHashChecked, + &result.LogicalHashChecked, + &result.CompressedBlocksChecked, + ); err != nil { + return VerifyResult{}, err + } + + var legacyBlocks int64 + if err := dbconn.QueryRowContext(ctx, ` + SELECT COUNT(*) + FROM blocks b + WHERE NOT EXISTS (SELECT 1 FROM chunk_block_refs r WHERE r.chunk_id = b.chunk_id) + `).Scan(&legacyBlocks); err != nil { + return VerifyResult{}, err + } + result.BlocksChecked += legacyBlocks + return result, nil +} + +func countVerifyResultForFile(ctx context.Context, dbconn *sql.DB, fileID int64) (VerifyResult, error) { + var result VerifyResult + if err := dbconn.QueryRowContext(ctx, ` + WITH target_blocks AS ( + SELECT DISTINCT sb.id, sb.physical_hash, sb.compressed_hash, sb.block_hash, sb.compression_codec + FROM file_chunk fc + JOIN chunk_block_refs r ON r.chunk_id = fc.chunk_id + JOIN storage_blocks sb ON sb.id = r.block_id + JOIN container c ON c.id = sb.container_id + WHERE fc.logical_file_id = $1 + AND c.quarantine = FALSE + ) + SELECT + COUNT(*), + COALESCE(SUM(CASE WHEN physical_hash IS NOT NULL AND length(physical_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN compressed_hash IS NOT NULL AND length(compressed_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN block_hash IS NOT NULL AND length(block_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN lower(trim(COALESCE(compression_codec, 'none'))) != 'none' THEN 1 ELSE 0 END), 0) + FROM target_blocks + `, fileID).Scan( + &result.BlocksChecked, + &result.PhysicalHashChecked, + &result.CompressedHashChecked, + &result.LogicalHashChecked, + &result.CompressedBlocksChecked, + ); err != nil { + return VerifyResult{}, err + } + + var legacyBlocks int64 + if err := dbconn.QueryRowContext(ctx, ` + SELECT COUNT(*) + FROM file_chunk fc + JOIN blocks b ON b.chunk_id = fc.chunk_id + WHERE fc.logical_file_id = $1 + AND NOT EXISTS (SELECT 1 FROM chunk_block_refs r WHERE r.chunk_id = b.chunk_id) + `, fileID).Scan(&legacyBlocks); err != nil { + return VerifyResult{}, err + } + result.BlocksChecked += legacyBlocks + return result, nil +} From 0d3b69ba75be4a9ce6a6ff5ba2e6907c306f4bc0 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 12:46:32 +0200 Subject: [PATCH 05/27] catalog: replace provisional planning contracts --- docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ....12-phase4-catalog-contract-replacement.md | 57 +++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 15 +- internal/catalog/catalog.go | 40 +-- internal/catalog/contracts.go | 78 ++++++ internal/catalog/contracts_test.go | 101 ++++++++ internal/catalog/errors.go | 91 ++++++- internal/catalog/gc_plan.go | 4 +- internal/catalog/graph.go | 5 +- internal/catalog/neutrality_test.go | 167 ++++-------- internal/catalog/placement.go | 6 +- internal/catalog/restore_plan.go | 5 +- internal/catalog/types.go | 240 +++++++++++------- 15 files changed, 559 insertions(+), 264 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md create mode 100644 internal/catalog/contracts.go create mode 100644 internal/catalog/contracts_test.go diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 9ad94870..46e67678 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–3 are complete; Phase 4 is next. + 0–4 are complete; Phase 5 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index ed2aa6d0..26d896ef 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -33,11 +33,13 @@ ## Phase 4 — Catalog Contract Replacement -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 4 catalog contracts](v1.13.12-phase4-catalog-contract-replacement.md). ## Phase 5 — Snapshot Graph Implementation -**Status:** Not started +**Status:** Next ## Phase 6 — Placement Implementation diff --git a/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md b/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md new file mode 100644 index 00000000..9d53cf5a --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md @@ -0,0 +1,57 @@ +# Coldkeep v1.13.12 Phase 4 — Catalog Contract Replacement + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `4 — Catalog Contract Replacement` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Contract replacement + +The four provisional planning shapes are replaced with complete neutral +contracts. This phase defines the contracts and their invariants only; the +four service methods retain the typed transitional `ErrNotImplemented` result +until their implementation phases. + +- Snapshot graph results contain deterministically ordered nodes, roots, child + relations, and an explicit none/present/missing parent state. A missing + historical parent cannot be silently converted into a root or invented edge. +- Chunk placements are ordered recipe entries using a strict legacy/packed + tagged union. Both variants carry bounded container metadata; legacy entries + carry their block descriptor, while packed entries carry transforms, sizes, + integrity hashes, and segment coordinates needed by the existing read path. +- Restore inputs use an explicit exclusive selector for logical file ID, + current stored path, or snapshot path. Restore results preserve logical + recipe provenance, exact nullable source metadata, and ordered placements. +- GC plans contain deterministically ordered logical-file roots with their + current/snapshot provenance and ordered protected snapshot metadata. They do + not contain storage deletion instructions. + +The catalog now has stable typed error codes and operation/invariant context. +Selector validation, GC exclusion normalization, and placement-union validation +are backend-neutral reusable contract functions. Catalog services never open or +commit a transaction; constructing a service with a caller-owned transaction +keeps multi-query plan construction inside that transaction. + +## Compatibility boundary + +No query, schema, migration, persisted row, storage byte, CLI, JSON, exit code, +transaction, lock, or backend default changed. Existing implemented catalog +lookups retain their historical projections. The planning methods still return +no partial result and are not production-consumable until Phases 5–9. + +## Validation + +- `go test -count=1 ./internal/catalog`: pass. +- `go test -race -count=1 ./internal/catalog`: pass. +- `go test -count=1 ./internal/engine`: pass after the catalog contract change. +- Release-state validator: development state, 1.13.12, zero violations. +- Recursive contract-graph neutrality check: pass; no database, storage, + domain, CLI, renderer, interface, or `any` field is exposed. +- Restore selector, GC normalization, placement union/bounds, and typed-error + tests: pass. +- Validation root: `/tmp/coldkeep-v11312-phase4.1E22St/` with isolated Go + caches and catalog storage root. + +Phase 5 is authorized to implement deterministic snapshot graph loading for +SQLite and PostgreSQL and to adopt that graph in lineage planning. It may not +implement placement, restore, or GC production adoption. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index c56dbaa1..aab8d57b 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–3 are Complete. Phase 4 is Next: replace the provisional snapshot -graph, placement, restore-plan, and GC-plan catalog shapes with complete -neutral contracts before implementing their queries. +Phases 0–4 are Complete. Phase 5 is Next: implement deterministic snapshot +graph loading for SQLite and PostgreSQL and adopt the graph in lineage +planning without moving rendering into the catalog. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 4330b480..383b86d6 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -59,11 +59,22 @@ ## Phase 4 — Catalog Contract Replacement -**Phase status:** Next +**Phase status:** Complete + +- [x] Snapshot graph contract distinguishes roots, present parents, and missing + historical parents with deterministic node/child ordering. +- [x] Placement contract is a strict legacy/packed union containing bounded + container, transform, hash, and segment metadata required by restore. +- [x] Restore selector exclusivity and caller-owned transaction semantics are + explicit and directly tested. +- [x] GC roots preserve current/snapshot provenance and deterministic ordering. +- [x] Catalog errors carry stable codes plus operation/invariant context. +- [x] Recursive neutrality checks reject backend, lower-layer, renderer, + interface, and `any` exposure. ## Phase 5 — Snapshot Graph Implementation -**Phase status:** Not started +**Phase status:** Next ## Phase 6 — Placement Implementation diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 94b0f78b..c7b34291 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -39,14 +39,8 @@ package catalog import ( "context" "database/sql" - "errors" ) -// ErrNotImplemented is returned by catalog methods whose metadata planning is -// intentionally deferred to a later migration phase. Production paths must not -// depend on these methods until they are implemented and tested. -var ErrNotImplemented = errors.New("catalog operation not implemented") - // DB is the minimal database abstraction the catalog depends on internally. // Both *sql.DB and *sql.Tx satisfy it, which keeps the facade testable and // transaction-aware without leaking a concrete handle to callers. @@ -85,8 +79,11 @@ type SnapshotCatalog interface { // SnapshotGraphCatalog exposes the snapshot lineage graph. // -// Deferred to Phase 5/6 (snapshot/GC migration). LoadSnapshotGraph returns -// ErrNotImplemented until then. +// Phase 5 implements this contract. Until then it returns the typed +// ErrNotImplemented sentinel and no partial graph. +// Empty catalogs return empty ordered slices. Historical missing parents are +// represented by SnapshotParentMissing; malformed cycles return a typed +// invariant_violation. The catalog never invents or silently repairs edges. type SnapshotGraphCatalog interface { LoadSnapshotGraph(ctx context.Context) (*SnapshotGraph, error) } @@ -100,28 +97,37 @@ type ReachabilityCatalog interface { // PlacementCatalog exposes chunk/block/container placement metadata. // -// Deferred to Phase 7/8 (restore/store migration). LoadChunkPlacements must -// represent both packed (storage_blocks/chunk_block_refs) and legacy (blocks) -// roots; that duality is migrated alongside restore/store. Returns -// ErrNotImplemented until then. +// Phase 6 implements this contract. It represents packed and legacy roots as a +// strict tagged union. Until then it returns no partial placement result. +// A missing logical file returns not_found. A zero-length logical file returns +// an empty placement slice. Missing, duplicate, mixed, or malformed placement +// rows return invariant_violation rather than a partial recipe. type PlacementCatalog interface { LoadChunkPlacements(ctx context.Context, logicalFileID int64) ([]ChunkPlacementRef, error) } // RestorePlanCatalog exposes restore-plan metadata. // -// Deferred to Phase 7 (restore migration), where the "restore must not write +// Phase 7 implements this contract, where the "restore must not write // outside destination" invariant is enforced at the engine/catalog boundary. -// Returns ErrNotImplemented until then. +// Until then it returns no partial restore plan. +// The service never opens or commits a transaction: constructing it with the +// caller's *sql.Tx keeps selector resolution and recipe loading in that exact +// transaction. +// A missing target returns not_found; a non-exclusive selector returns +// invalid_argument; ambiguous or incomplete metadata returns conflict or +// invariant_violation. No partial plan is returned on any error. type RestorePlanCatalog interface { LoadRestorePlanMetadata(ctx context.Context, input RestorePlanInput) (*RestorePlanMetadata, error) } // GCPlanCatalog exposes GC-plan metadata. // -// Deferred to Phase 6 (GC migration), where the "GC must never delete -// reachable data" invariant is tested at the engine/catalog boundary. Returns -// ErrNotImplemented until then. +// Phase 9 implements and adopts this contract, where the "GC must never delete +// reachable data" invariant is tested at the engine/catalog boundary. Until +// then it returns no partial GC plan. +// Excluded snapshot IDs are validated before reads; missing IDs return a typed +// not_found error, while malformed graph rows return invariant_violation. type GCPlanCatalog interface { LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*GCPlanMetadata, error) } diff --git a/internal/catalog/contracts.go b/internal/catalog/contracts.go new file mode 100644 index 00000000..b9a8b4bc --- /dev/null +++ b/internal/catalog/contracts.go @@ -0,0 +1,78 @@ +package catalog + +import ( + "fmt" + "sort" + "strings" +) + +func ValidateRestorePlanInput(input RestorePlanInput) error { + valid := false + switch input.Selector { + case RestoreByFileID: + valid = input.FileID > 0 && input.StoredPath == "" && input.SnapshotID == "" && input.SnapshotPath == "" + case RestoreByStoredPath: + valid = input.FileID == 0 && input.StoredPath != "" && input.SnapshotID == "" && input.SnapshotPath == "" + case RestoreBySnapshotPath: + valid = input.FileID == 0 && input.StoredPath == "" && input.SnapshotID != "" && input.SnapshotPath != "" + } + if valid { + return nil + } + return NewError(ErrorInvalidArgument, "load restore plan", "exactly_one_restore_selector", "restore selector must contain exactly one valid file_id, stored_path, or snapshot_path target", nil) +} + +// NormalizeGCPlanInput validates, deduplicates, and sorts IDs without mutating input. +func NormalizeGCPlanInput(input GCPlanInput) (GCPlanInput, error) { + seen := make(map[string]struct{}, len(input.ExcludeSnapshotIDs)) + for _, id := range input.ExcludeSnapshotIDs { + if strings.TrimSpace(id) == "" { + return GCPlanInput{}, NewError(ErrorInvalidArgument, "load GC plan", "nonempty_snapshot_id", "excluded snapshot ID must not be empty", nil) + } + seen[id] = struct{}{} + } + ids := make([]string, 0, len(seen)) + for id := range seen { + ids = append(ids, id) + } + sort.Strings(ids) + return GCPlanInput{ExcludeSnapshotIDs: ids}, nil +} + +// ValidateChunkPlacement enforces the tagged union and shared range invariants. +func ValidateChunkPlacement(placement ChunkPlacementRef) error { + if placement.ChunkOrder < 0 || placement.ChunkID <= 0 || placement.ChunkHash == "" || placement.ChunkSize <= 0 || placement.ChunkerVersion == "" || placement.ChunkStatus == "" { + return invalidPlacement(placement, "chunk identity and order must be complete") + } + switch placement.Kind { + case PlacementLegacy: + if placement.Legacy == nil || placement.Packed != nil { + return invalidPlacement(placement, "legacy placement must contain only legacy metadata") + } + if placement.Legacy.BlockID <= 0 || placement.Legacy.Codec == "" || placement.Legacy.FormatVersion <= 0 || placement.Legacy.PlaintextSize <= 0 || placement.Legacy.StoredSize <= 0 || !validContainerRead(placement.Legacy.Container, placement.Legacy.ContainerOffset, placement.Legacy.StoredSize) { + return invalidPlacement(placement, "legacy block and container bounds must be valid") + } + case PlacementPacked: + if placement.Packed == nil || placement.Legacy != nil { + return invalidPlacement(placement, "packed placement must contain only packed metadata") + } + if placement.Packed.BlockID <= 0 || placement.Packed.Codec == "" || placement.Packed.FormatVersion <= 0 || placement.Packed.PlaintextSize <= 0 || placement.Packed.CompressionCodec == "" || placement.Packed.StoredSize <= 0 || len(placement.Packed.BlockHash) == 0 || placement.Packed.OffsetInBlock < 0 || placement.Packed.SizeInBlock <= 0 || placement.Packed.OffsetInBlock+placement.Packed.SizeInBlock > placement.Packed.PlaintextSize || !validContainerRead(placement.Packed.Container, placement.Packed.ContainerOffset, placement.Packed.StoredSize) { + return invalidPlacement(placement, "packed block, segment, and container bounds must be valid") + } + default: + return invalidPlacement(placement, "placement kind must be legacy or packed") + } + return nil +} + +func validContainerRead(container ContainerPlacementRef, offset, size int64) bool { + if container.ID <= 0 || container.Filename == "" || container.Quarantined || offset < 0 || size <= 0 || container.CurrentSize < 0 || container.MaxSize <= 0 { + return false + } + end := offset + size + return end >= offset && end <= container.CurrentSize && end <= container.MaxSize +} + +func invalidPlacement(placement ChunkPlacementRef, message string) error { + return NewError(ErrorInvariantViolation, "load chunk placements", "exactly_one_valid_placement_per_chunk", fmt.Sprintf("chunk %d: %s", placement.ChunkID, message), nil) +} diff --git a/internal/catalog/contracts_test.go b/internal/catalog/contracts_test.go new file mode 100644 index 00000000..a03cbdc1 --- /dev/null +++ b/internal/catalog/contracts_test.go @@ -0,0 +1,101 @@ +package catalog_test + +import ( + "errors" + "reflect" + "testing" + + "github.com/franchoy/coldkeep/internal/catalog" +) + +func TestRestorePlanSelectorContract(t *testing.T) { + valid := []catalog.RestorePlanInput{ + {Selector: catalog.RestoreByFileID, FileID: 7}, + {Selector: catalog.RestoreByStoredPath, StoredPath: "folder/file.txt"}, + {Selector: catalog.RestoreBySnapshotPath, SnapshotID: "snap-1", SnapshotPath: "folder/file.txt"}, + } + for _, input := range valid { + if err := catalog.ValidateRestorePlanInput(input); err != nil { + t.Errorf("valid input %+v: %v", input, err) + } + } + + invalid := []catalog.RestorePlanInput{ + {}, + {Selector: catalog.RestoreByFileID}, + {Selector: catalog.RestoreByFileID, FileID: 1, StoredPath: "also-set"}, + {Selector: catalog.RestoreByStoredPath, FileID: 1, StoredPath: "file"}, + {Selector: catalog.RestoreBySnapshotPath, SnapshotID: "snap"}, + {Selector: "future", FileID: 1}, + } + for _, input := range invalid { + err := catalog.ValidateRestorePlanInput(input) + if !catalog.IsCode(err, catalog.ErrorInvalidArgument) { + t.Errorf("invalid input %+v: got %v", input, err) + } + var typed *catalog.Error + if !errors.As(err, &typed) || typed.Invariant != "exactly_one_restore_selector" { + t.Errorf("invalid input %+v lacks selector invariant: %v", input, err) + } + } +} + +func TestGCPlanInputNormalizationIsSortedUniqueAndNonMutating(t *testing.T) { + input := catalog.GCPlanInput{ExcludeSnapshotIDs: []string{"z", "a", "z", "m"}} + got, err := catalog.NormalizeGCPlanInput(input) + if err != nil { + t.Fatal(err) + } + if want := []string{"a", "m", "z"}; !reflect.DeepEqual(got.ExcludeSnapshotIDs, want) { + t.Fatalf("got %v want %v", got.ExcludeSnapshotIDs, want) + } + if want := []string{"z", "a", "z", "m"}; !reflect.DeepEqual(input.ExcludeSnapshotIDs, want) { + t.Fatalf("input mutated: %v", input.ExcludeSnapshotIDs) + } + if _, err := catalog.NormalizeGCPlanInput(catalog.GCPlanInput{ExcludeSnapshotIDs: []string{"ok", " "}}); !catalog.IsCode(err, catalog.ErrorInvalidArgument) { + t.Fatalf("empty ID error: %v", err) + } +} + +func TestChunkPlacementTaggedUnionContract(t *testing.T) { + container := catalog.ContainerPlacementRef{ID: 2, Filename: "c-2", CurrentSize: 4096, MaxSize: 4096} + base := catalog.ChunkPlacementRef{ChunkOrder: 0, ChunkID: 3, ChunkHash: "abc", ChunkSize: 9, ChunkerVersion: "v2-fastcdc", ChunkStatus: "COMPLETED"} + legacy := base + legacy.Kind = catalog.PlacementLegacy + legacy.Legacy = &catalog.LegacyChunkPlacement{BlockID: 4, Codec: "plain", FormatVersion: 1, PlaintextSize: 9, StoredSize: 9, Container: container} + if err := catalog.ValidateChunkPlacement(legacy); err != nil { + t.Fatalf("valid legacy: %v", err) + } + + packed := base + packed.Kind = catalog.PlacementPacked + packed.Packed = &catalog.PackedChunkPlacement{BlockID: 5, Codec: "none", FormatVersion: 1, PlaintextSize: 20, CompressionCodec: "none", StoredSize: 20, BlockHash: []byte{1}, Container: container, OffsetInBlock: 2, SizeInBlock: 9} + if err := catalog.ValidateChunkPlacement(packed); err != nil { + t.Fatalf("valid packed: %v", err) + } + + invalid := packed + invalid.Legacy = legacy.Legacy + if err := catalog.ValidateChunkPlacement(invalid); !catalog.IsCode(err, catalog.ErrorInvariantViolation) { + t.Fatalf("invalid union: %v", err) + } + invalid = packed + invalid.Packed.SizeInBlock = 0 + if err := catalog.ValidateChunkPlacement(invalid); !catalog.IsCode(err, catalog.ErrorInvariantViolation) { + t.Fatalf("invalid range: %v", err) + } +} + +func TestCatalogErrorPreservesStableContextAndCause(t *testing.T) { + cause := errors.New("backend detail") + err := catalog.NewError(catalog.ErrorConflict, "load graph", "acyclic_snapshot_graph", "graph conflict", cause) + if !errors.Is(err, cause) || !catalog.IsCode(err, catalog.ErrorConflict) { + t.Fatalf("typed error classification failed: %v", err) + } + if err.Operation != "load graph" || err.Invariant != "acyclic_snapshot_graph" { + t.Fatalf("typed context lost: %+v", err) + } + if !catalog.IsCode(catalog.ErrNotImplemented, catalog.ErrorUnsupported) || !catalog.IsDeferred(catalog.ErrNotImplemented) { + t.Fatalf("deferred sentinel not typed: %v", catalog.ErrNotImplemented) + } +} diff --git a/internal/catalog/errors.go b/internal/catalog/errors.go index 6a17f9fb..ad498f3a 100644 --- a/internal/catalog/errors.go +++ b/internal/catalog/errors.go @@ -1,14 +1,83 @@ package catalog -import "errors" - -// IsDeferred classifies catalog deferred surfaces represented by -// catalog.ErrNotImplemented. -// -// It is intentionally narrow for v1.13.2. It does not mean deferred catalog -// methods are implemented. It does not classify engine unsupported errors, -// validation errors, domain errors, invariant errors, runtime errors, or -// storage failures. The helper preserves errors.Is compatibility. -func IsDeferred(err error) bool { - return errors.Is(err, ErrNotImplemented) +import ( + "errors" + "fmt" +) + +type ErrorCode string + +const ( + ErrorInvalidArgument ErrorCode = "invalid_argument" + ErrorNotFound ErrorCode = "not_found" + ErrorUnsupported ErrorCode = "unsupported" + ErrorInvariantViolation ErrorCode = "invariant_violation" + ErrorConflict ErrorCode = "conflict" + ErrorCancelled ErrorCode = "cancelled" + ErrorOperationFailed ErrorCode = "operation_failed" +) + +// Error is the backend-neutral catalog error boundary. +type Error struct { + Code ErrorCode + Operation string + Invariant string + Message string + Cause error +} + +func (e *Error) Error() string { + if e == nil { + return "" + } + message := e.Message + if message == "" && e.Cause != nil { + message = e.Cause.Error() + } + if e.Operation == "" { + return message + } + if message == "" { + return "catalog " + e.Operation + } + return fmt.Sprintf("catalog %s: %s", e.Operation, message) +} + +func (e *Error) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +func NewError(code ErrorCode, operation, invariant, message string, cause error) *Error { + return &Error{Code: code, Operation: operation, Invariant: invariant, Message: message, Cause: cause} +} + +func CodeOf(err error) (ErrorCode, bool) { + var catalogErr *Error + if !errors.As(err, &catalogErr) || catalogErr == nil { + return "", false + } + return catalogErr.Code, true +} + +func IsCode(err error, code ErrorCode) bool { + actual, ok := CodeOf(err) + return ok && actual == code } + +var deferredCause = errors.New("catalog operation not implemented") + +// ErrNotImplemented is the typed transitional sentinel for the four planning +// methods. Production must not consume a method while it returns this value. +var ErrNotImplemented error = NewError( + ErrorUnsupported, + "deferred planning operation", + "catalog_planning_api_must_be_implemented", + deferredCause.Error(), + deferredCause, +) + +// IsDeferred recognizes only the transitional sentinel, not all unsupported errors. +func IsDeferred(err error) bool { return errors.Is(err, ErrNotImplemented) } diff --git a/internal/catalog/gc_plan.go b/internal/catalog/gc_plan.go index a7ee6a79..e2c02313 100644 --- a/internal/catalog/gc_plan.go +++ b/internal/catalog/gc_plan.go @@ -4,9 +4,7 @@ import "context" // LoadGCPlanMetadata implements GCPlanCatalog. // -// Deferred to Phase 6 (GC migration). The GC plan must be available before -// "GC must never delete reachable data" can be tested at the engine/catalog -// boundary. Callers must not rely on this method until it is implemented. +// Phase 9 implements and adopts deterministic reachability metadata. func (s *Service) LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*GCPlanMetadata, error) { _, _ = ctx, input return nil, ErrNotImplemented diff --git a/internal/catalog/graph.go b/internal/catalog/graph.go index 5a84213b..c1f89b97 100644 --- a/internal/catalog/graph.go +++ b/internal/catalog/graph.go @@ -4,10 +4,7 @@ import "context" // LoadSnapshotGraph implements SnapshotGraphCatalog. // -// Deferred to Phase 5/6 (snapshot/GC migration). The snapshot graph traversal -// depends on snapshot reachability semantics being fully understood and tested -// before it is expressed through a catalog contract. Callers must not rely on -// this method until it is implemented. +// Phase 5 implements the backend-neutral graph contract defined in types.go. func (s *Service) LoadSnapshotGraph(ctx context.Context) (*SnapshotGraph, error) { _ = ctx return nil, ErrNotImplemented diff --git a/internal/catalog/neutrality_test.go b/internal/catalog/neutrality_test.go index 00cb5fb0..451f0eb6 100644 --- a/internal/catalog/neutrality_test.go +++ b/internal/catalog/neutrality_test.go @@ -1,148 +1,73 @@ package catalog_test import ( - "database/sql" - "io" "reflect" + "strings" "testing" "github.com/franchoy/coldkeep/internal/catalog" ) -// TestCatalogExportedTypesAreNeutral verifies that exported catalog contract -// types do not expose any backend-specific or renderer-specific types in their -// fields. -// -// Forbidden field types: *sql.DB, *sql.Tx, sql.Rows, sql.Row, io.Writer, -// io.Reader, and any cobra/CLI renderer type. -func TestCatalogExportedTypesAreNeutral(t *testing.T) { - types := []any{ - catalog.LogicalFileRef{}, - catalog.PhysicalFileRef{}, - catalog.SnapshotRef{}, - catalog.SnapshotFilter{}, - catalog.SnapshotGraph{}, - catalog.ReachabilityRoots{}, - catalog.ChunkPlacementRef{}, - catalog.RestorePlanInput{}, - catalog.RestorePlanMetadata{}, - catalog.GCPlanInput{}, - catalog.GCPlanMetadata{}, +// TestCatalogContractGraphsAreNeutral walks every exported request/result +// graph. It rejects interface/any fields and dependencies on database, storage, +// domain, CLI, renderer, and stream implementations. +func TestCatalogContractGraphsAreNeutral(t *testing.T) { + roots := []any{ + catalog.LogicalFileRef{}, catalog.PhysicalFileRef{}, + catalog.SnapshotRef{}, catalog.SnapshotFilter{}, + catalog.SnapshotGraph{}, catalog.ReachabilityRoots{}, + catalog.ContainerPlacementRef{}, catalog.LegacyChunkPlacement{}, + catalog.PackedChunkPlacement{}, catalog.ChunkPlacementRef{}, + catalog.RestorePlanInput{}, catalog.RestoreLogicalFileRef{}, + catalog.RestoreSourceRef{}, catalog.RestorePlanMetadata{}, + catalog.GCPlanInput{}, catalog.GCReachabilityRoot{}, catalog.GCPlanMetadata{}, } - forbiddenTypes := []reflect.Type{ - reflect.TypeOf(&sql.DB{}), - reflect.TypeOf(&sql.Tx{}), - reflect.TypeOf(sql.Rows{}), - reflect.TypeOf(sql.Row{}), - reflect.TypeOf((*io.Writer)(nil)).Elem(), - reflect.TypeOf((*io.Reader)(nil)).Elem(), + forbiddenPackages := []string{ + "database/sql", "internal/storage", "internal/domain", "internal/db", + "cmd/coldkeep", "internal/cli", "github.com/spf13/cobra", "io", } - forbiddenNames := []string{ - "*sql.DB", "*sql.Tx", "sql.Rows", "sql.Row", - "sql.NullTime", // internal scan type must not leak into exported types - } - - for _, v := range types { - rt := reflect.TypeOf(v) - checkTypeNeutral(t, rt, forbiddenTypes, forbiddenNames, rt.Name()) + for _, root := range roots { + seen := make(map[reflect.Type]bool) + checkCatalogContractType(t, reflect.TypeOf(root), reflect.TypeOf(root).Name(), forbiddenPackages, seen) } } -func checkTypeNeutral(t *testing.T, rt reflect.Type, forbiddenTypes []reflect.Type, forbiddenNames []string, path string) { +func checkCatalogContractType(t *testing.T, typ reflect.Type, path string, forbiddenPackages []string, seen map[reflect.Type]bool) { t.Helper() - rt = dereferenceCatalogType(rt) - if !isStructType(rt) { - return - } - - for _, field := range exportedStructFields(rt) { - checkFieldNeutrality(t, field, forbiddenTypes, forbiddenNames, path) + for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice || typ.Kind() == reflect.Array || typ.Kind() == reflect.Map { + if typ.Kind() == reflect.Map { + checkCatalogContractType(t, typ.Key(), path+".", forbiddenPackages, seen) + } + typ = typ.Elem() } -} - -func checkFieldNeutrality( - t *testing.T, - field reflect.StructField, - forbiddenTypes []reflect.Type, - forbiddenNames []string, - parentPath string, -) { - t.Helper() - fieldPath := parentPath + "." + field.Name - fieldType := field.Type - - checkForbiddenFieldTypes(t, fieldPath, fieldType, forbiddenTypes) - checkForbiddenFieldNames(t, fieldPath, fieldType, forbiddenNames) - recurseIntoNestedStruct(t, fieldPath, fieldType, forbiddenTypes, forbiddenNames) -} - -func dereferenceCatalogType(rt reflect.Type) reflect.Type { - if rt.Kind() == reflect.Ptr { - return rt.Elem() + if seen[typ] { + return } - return rt -} + seen[typ] = true -func isStructType(rt reflect.Type) bool { - return rt.Kind() == reflect.Struct -} - -func exportedStructFields(rt reflect.Type) []reflect.StructField { - fields := make([]reflect.StructField, 0, rt.NumField()) - for i := 0; i < rt.NumField(); i++ { - field := rt.Field(i) - if field.IsExported() { - fields = append(fields, field) - } + if typ.Kind() == reflect.Interface { + t.Errorf("%s exposes interface/any type %v", path, typ) + return } - return fields -} - -func checkForbiddenFieldTypes(t *testing.T, fieldPath string, fieldType reflect.Type, forbiddenTypes []reflect.Type) { - t.Helper() - for _, forbidden := range forbiddenTypes { - if isForbiddenFieldType(fieldType, forbidden) { - t.Errorf("field %s has forbidden type %v", fieldPath, forbidden) - } + if typ.Kind() == reflect.Func || typ.Kind() == reflect.Chan || typ.Kind() == reflect.UnsafePointer { + t.Errorf("%s exposes executable or unsafe type %v", path, typ) + return } -} - -func isForbiddenFieldType(fieldType, forbidden reflect.Type) bool { - return fieldType == forbidden || implementsForbiddenInterface(fieldType, forbidden) -} - -func implementsForbiddenInterface(fieldType, forbidden reflect.Type) bool { - return forbidden.Kind() == reflect.Interface && fieldType.Implements(forbidden) -} - -func checkForbiddenFieldNames(t *testing.T, fieldPath string, fieldType reflect.Type, forbiddenNames []string) { - t.Helper() - for _, name := range forbiddenNames { - if fieldType.String() == name { - t.Errorf("field %s has forbidden type %q", fieldPath, name) + for _, forbidden := range forbiddenPackages { + if typ.PkgPath() == forbidden || strings.Contains(typ.PkgPath(), forbidden) { + t.Errorf("%s exposes forbidden package type %v", path, typ) + return } } -} - -func recurseIntoNestedStruct( - t *testing.T, - fieldPath string, - fieldType reflect.Type, - forbiddenTypes []reflect.Type, - forbiddenNames []string, -) { - t.Helper() - if nestedStructType(fieldType) == nil { + if typ.Kind() != reflect.Struct || (typ.PkgPath() != "" && typ.PkgPath() != "github.com/franchoy/coldkeep/internal/catalog") { return } - checkTypeNeutral(t, fieldType, forbiddenTypes, forbiddenNames, fieldPath) -} - -func nestedStructType(fieldType reflect.Type) reflect.Type { - fieldType = dereferenceCatalogType(fieldType) - if isStructType(fieldType) { - return fieldType + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if !field.IsExported() { + continue + } + checkCatalogContractType(t, field.Type, path+"."+field.Name, forbiddenPackages, seen) } - return nil } diff --git a/internal/catalog/placement.go b/internal/catalog/placement.go index a3bc4268..7cd0e2c5 100644 --- a/internal/catalog/placement.go +++ b/internal/catalog/placement.go @@ -4,11 +4,7 @@ import "context" // LoadChunkPlacements implements PlacementCatalog. // -// Deferred to Phase 7/8 (restore/store migration). Placement metadata must -// unify packed (storage_blocks / chunk_block_refs) and legacy (blocks) roots -// without privileging either representation. That duality is safely expressible -// only after the restore and store migrations are underway. Callers must not -// rely on this method until it is implemented. +// Phase 6 implements the strict packed/legacy union defined in types.go. func (s *Service) LoadChunkPlacements(ctx context.Context, logicalFileID int64) ([]ChunkPlacementRef, error) { _, _ = ctx, logicalFileID return nil, ErrNotImplemented diff --git a/internal/catalog/restore_plan.go b/internal/catalog/restore_plan.go index 2e9951ba..e30d5c1d 100644 --- a/internal/catalog/restore_plan.go +++ b/internal/catalog/restore_plan.go @@ -4,10 +4,7 @@ import "context" // LoadRestorePlanMetadata implements RestorePlanCatalog. // -// Deferred to Phase 7 (restore migration). The restore plan must be available -// before "restore must not write outside the intended destination" can be -// enforced at the engine/catalog boundary. Callers must not rely on this -// method until it is implemented. +// Phase 7 implements selector resolution and immutable recipe construction. func (s *Service) LoadRestorePlanMetadata(ctx context.Context, input RestorePlanInput) (*RestorePlanMetadata, error) { _, _ = ctx, input return nil, ErrNotImplemented diff --git a/internal/catalog/types.go b/internal/catalog/types.go index f81a2d03..4d23f3e0 100644 --- a/internal/catalog/types.go +++ b/internal/catalog/types.go @@ -2,136 +2,194 @@ package catalog import "time" -// LogicalFileRef is the catalog's neutral representation of a logical_file row. -// Field names and types are aligned with the logical_file table schema. type LogicalFileRef struct { ID int64 OriginalName string TotalSize int64 FileHash string RefCount int - Status string // "PROCESSING" | "COMPLETED" | "ABORTED" + Status string } -// PhysicalFileRef is the catalog's neutral representation of a physical_file -// row. Path is the primary key in the schema. -// MTime is nil when the mtime column is NULL; Mode is zero when NULL. -// IsMetadataComplete maps to is_metadata_complete (bool/INTEGER 0-1). +// PhysicalFileRef preserves the historical lookup projection: Mode is zero +// when the database value is NULL. RestoreSourceRef preserves exact nullness. type PhysicalFileRef struct { Path string LogicalFileID int64 Mode int - MTime *time.Time // nil when NULL in DB + MTime *time.Time IsMetadataComplete bool } -// SnapshotRef is the catalog's neutral representation of a snapshot row. -// Type is "full" or "partial". ParentID and Label are empty when NULL in the DB. type SnapshotRef struct { ID string - Type string // "full" | "partial" + Type string Label string ParentID string CreatedAt time.Time } -// SnapshotFilter constrains the result set of ListSnapshots. Zero values mean -// "no filter on this dimension". type SnapshotFilter struct { - // Type filters by snapshot type; empty means all. - Type string - // LabelSubstring filters snapshots whose label contains the substring. + Type string LabelSubstring string - // Since and Until bound the created_at range (inclusive). - Since *time.Time - Until *time.Time - // Limit caps the result count when greater than zero. - Limit int + Since *time.Time + Until *time.Time + Limit int } -// SnapshotGraph is a renderer-neutral representation of the snapshot lineage. -// Deferred to Phase 5/6; fields are minimal so the skeleton compiles without -// committing to the graph traversal shape. +// SnapshotParentState distinguishes roots from historical missing-parent rows. +// Missing parents never create invented edges and do not turn children into roots. +type SnapshotParentState string + +const ( + SnapshotParentNone SnapshotParentState = "none" + SnapshotParentPresent SnapshotParentState = "present" + SnapshotParentMissing SnapshotParentState = "missing" +) + +// SnapshotGraphNode contains child IDs ordered by created_at, then ID. +type SnapshotGraphNode struct { + Snapshot SnapshotRef + ParentState SnapshotParentState + ChildIDs []string +} + +// SnapshotGraph nodes and roots are ordered by created_at ascending, then ID. type SnapshotGraph struct { - // Roots are the snapshot IDs with no parent. - Roots []string - // Edges maps snapshot ID to parent snapshot ID. - Edges map[string]string -} - -// ReachabilityRoots holds the logical file ID sets used by GC and verification. -// Current is the set reachable from physical_file (active working set); -// Snapshot is the set reachable from snapshot_file (snapshot-protected). -// -// Both packed and legacy storage roots contribute to the same reachability sets -// because reachability is expressed in terms of logical file IDs, not storage -// format. + Nodes []SnapshotGraphNode + RootIDs []string +} + +// ReachabilityRoots is the compatibility shape used by existing callers. type ReachabilityRoots struct { - // Current is the set of logical file IDs referenced by physical_file rows. - Current map[int64]struct{} - // Snapshot is the set of logical file IDs referenced by snapshot_file rows. + Current map[int64]struct{} Snapshot map[int64]struct{} } -// ChunkPlacementRef describes where a single chunk is stored. -// It covers both packed storage (storage_blocks/chunk_block_refs) and legacy -// storage (blocks). The Packed field distinguishes the two paths. -// -// Deferred to Phase 7/8. +type PlacementKind string + +const ( + PlacementLegacy PlacementKind = "legacy" + PlacementPacked PlacementKind = "packed" +) + +// ContainerPlacementRef contains the facts required for bounded reads. +type ContainerPlacementRef struct { + ID int64 + Filename string + Sealed bool + Sealing bool + ContainerHash string + Quarantined bool + CurrentSize int64 + MaxSize int64 +} + +type LegacyChunkPlacement struct { + BlockID int64 + Codec string + FormatVersion int + PlaintextSize int64 + StoredSize int64 + Nonce []byte + Container ContainerPlacementRef + ContainerOffset int64 +} + +type PackedChunkPlacement struct { + BlockID int64 + FormatVersion int + Codec string + PlaintextSize int64 + CompressionCodec string + CompressionLevel *int + CompressedSize *int64 + StoredSize int64 + BlockHash []byte + CompressedHash []byte + PhysicalHash []byte + Container ContainerPlacementRef + ContainerOffset int64 + OffsetInBlock int64 + SizeInBlock int64 +} + +// ChunkPlacementRef is one ordered logical-file recipe entry. Kind and the +// placement pointers form a strict tagged union. Results are contiguous from 0. type ChunkPlacementRef struct { - ChunkID int64 - ChunkHash string - ContainerID int64 - // BlockID is storage_blocks.id (packed) or blocks.id (legacy). - BlockID int64 - Offset int64 // container_offset (packed) or block_offset (legacy) - Size int64 // stored_size - OffsetInBlock int64 // offset_in_block (packed only; zero for legacy) - SizeInBlock int64 // size_in_block (packed only; zero for legacy) - Packed bool // true=packed (storage_blocks); false=legacy (blocks) -} - -// RestorePlanInput selects a restore target. Exactly one of FileID, -// SnapshotID, or StoredPath should be populated for a given lookup. -// -// Deferred to Phase 7 (restore migration). + ChunkOrder int64 + ChunkID int64 + ChunkHash string + ChunkSize int64 + ChunkerVersion string + ChunkStatus string + Kind PlacementKind + Legacy *LegacyChunkPlacement + Packed *PackedChunkPlacement +} + +type RestoreSelector string + +const ( + RestoreByFileID RestoreSelector = "file_id" + RestoreByStoredPath RestoreSelector = "stored_path" + RestoreBySnapshotPath RestoreSelector = "snapshot_path" +) + +// RestorePlanInput is a strict selector union: FileID; StoredPath; or +// SnapshotID plus SnapshotPath, selected by Selector. type RestorePlanInput struct { - FileID int64 - SnapshotID string - StoredPath string -} - -// RestorePlanMetadata is the catalog-level resolution of a restore target. -// -// Safety note: "restore must not write outside the intended destination" will -// be enforced by the engine using this metadata. The catalog provides complete -// and accurate metadata; the engine enforces destination safety. -// -// Deferred to Phase 7. + Selector RestoreSelector + FileID int64 + StoredPath string + SnapshotID string + SnapshotPath string +} + +type RestoreLogicalFileRef struct { + ID int64 + OriginalName string + TotalSize int64 + FileHash string + ChunkerVersion string + Status string +} + +// RestoreSourceRef preserves nullable current-file and snapshot metadata. +type RestoreSourceRef struct { + StoredPath string + SnapshotID string + SnapshotPath string + Mode *int64 + MTime *time.Time + UID *int64 + GID *int64 + IsMetadataComplete bool +} + +// RestorePlanMetadata is a complete immutable recipe; payload I/O stays below +// the catalog boundary. type RestorePlanMetadata struct { - LogicalFile LogicalFileRef - PhysicalFiles []PhysicalFileRef - Placements []ChunkPlacementRef + Selector RestorePlanInput + LogicalFile RestoreLogicalFileRef + Source RestoreSourceRef + Placements []ChunkPlacementRef } -// GCPlanInput parameterises a GC-plan metadata query. ExcludeSnapshotIDs lists -// snapshot IDs excluded from the reachability roots (e.g. being deleted). -// -// Deferred to Phase 6 (GC migration). type GCPlanInput struct { ExcludeSnapshotIDs []string } -// GCPlanMetadata is the catalog-level metadata needed to plan a GC run. -// Reachability is expressed in terms of logical file IDs so the engine can -// compute chunk/block/container deletion lists without knowing the storage -// format. Both packed and legacy roots are included. -// -// Deferred to Phase 6. +// GCReachabilityRoot records all reasons a logical file remains protected. +type GCReachabilityRoot struct { + LogicalFileID int64 + Current bool + SnapshotIDs []string +} + +// GCPlanMetadata contains deterministic mark roots, never sweep instructions. +// Roots are ordered by logical ID; snapshots by created_at, then ID. type GCPlanMetadata struct { - // ReachableLogicalFileIDs is the set of all retained logical file IDs. - ReachableLogicalFileIDs map[int64]struct{} - // ProtectedSnapshotIDs is the set of snapshot IDs that contribute to - // reachability. - ProtectedSnapshotIDs []string + Roots []GCReachabilityRoot + ProtectedSnapshots []SnapshotRef } From bdb6d1e5bea718ca23301e554f98a839aacb0f5a Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 12:59:07 +0200 Subject: [PATCH 06/27] catalog: implement snapshot graph planning --- cmd/coldkeep/main.go | 2 + docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...12-phase5-snapshot-graph-implementation.md | 72 ++++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 13 +- internal/catalog/backend_contract_test.go | 52 +++++- internal/catalog/graph.go | 168 +++++++++++++++++- internal/catalog/graph_test.go | 72 ++++++++ internal/catalog/service_test.go | 15 +- internal/engine/candidates.go | 42 +++-- internal/engine/default_engine.go | 68 ++++++- internal/engine/errors.go | 26 ++- internal/engine/snapshot_engine_test.go | 64 +++++++ ...snapshot_selector_backend_contract_test.go | 10 ++ internal/snapshot/snapshot.go | 3 + internal/testutil/backendtest/backendtest.go | 2 + 17 files changed, 577 insertions(+), 46 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase5-snapshot-graph-implementation.md create mode 100644 internal/catalog/graph_test.go diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 86b3d39b..190e0af9 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -377,6 +377,7 @@ var listSnapshotsPhase = func(ctx context.Context, db *sql.DB, filter snapshot.S Since: filter.Since, Until: filter.Until, Limit: filter.Limit, + Tree: filter.Tree, } if filter.Type != nil { req.Type = engine.SnapshotType(*filter.Type) @@ -5711,6 +5712,7 @@ func runSnapshotListCommand(parsed parsedCommandLine, outputMode cliOutputMode) treeMode := parsed.hasFlag("tree") filter := snapshot.SnapshotListFilter{} + filter.Tree = treeMode if value, ok := parsed.lastFlagValue("type"); ok { trimmed := strings.ToLower(strings.TrimSpace(value)) if trimmed != "full" && trimmed != "partial" { diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 46e67678..ea2a0c3f 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–4 are complete; Phase 5 is next. + 0–5 are complete; Phase 6 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 26d896ef..7b9d5b05 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -39,11 +39,13 @@ ## Phase 5 — Snapshot Graph Implementation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 5 snapshot graph](v1.13.12-phase5-snapshot-graph-implementation.md). ## Phase 6 — Placement Implementation -**Status:** Not started +**Status:** Next ## Phase 7 — Restore-Plan Implementation diff --git a/docs/release/v1.13/v1.13.12-phase5-snapshot-graph-implementation.md b/docs/release/v1.13/v1.13.12-phase5-snapshot-graph-implementation.md new file mode 100644 index 00000000..a4fa38c6 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase5-snapshot-graph-implementation.md @@ -0,0 +1,72 @@ +# Coldkeep v1.13.12 Phase 5 — Snapshot Graph Implementation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `5 — Snapshot Graph Implementation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Implementation + +`catalog.Service.LoadSnapshotGraph` now performs one read through the injected +database or transaction and returns a deterministic neutral graph. Ordering is +enforced in Go by `created_at` ascending and ID ascending, independent of +backend collation. Children inherit that order. + +Parent metadata is classified explicitly: + +- `none` for a true root; +- `present` for a resolved parent and real edge; +- `missing` for a historical parent ID that no longer resolves. + +Missing parents remain visible without an invented edge. Empty or duplicate +IDs, malformed timestamps, self-cycles, and multi-node cycles fail closed with +typed catalog invariant errors and no partial graph. Cancellation and database +failures retain their error chain under stable catalog codes. + +`Engine.SnapshotList` consumes this graph when `Tree` is requested, projects it +through engine-owned DTOs, and translates catalog error/invariant context into +the stable engine taxonomy. The filtered list and complete graph are read in +one engine-owned read-only transaction, while the catalog remains transaction +lifecycle neutral. The compatibility adapter now carries the existing CLI +`--tree` flag into that engine operation. Snapshot list order remains newest +first; graph nodes and child relationships remain oldest first for lineage +planning. Tree line rendering remains in the CLI. + +## Compatibility + +No schema, migration, snapshot row, storage byte, CLI syntax, JSON shape, text +line, exit code, transaction, lock, or backend default changed. Non-tree lists +retain their existing list query. Tree requests add a read-only catalog graph +query but preserve the historical output ordering and missing-parent display. + +## Validation + +- `go test -count=1 ./internal/catalog`: pass. +- `go test -race -count=1 ./internal/catalog`: pass. +- `go test -count=1 ./internal/engine`: pass. +- `go test -count=1 ./internal/snapshot`: pass. +- `go test -count=1 ./cmd/coldkeep`: pass. +- Shared graph contract on SQLite and PostgreSQL: pass. +- Production `Engine.SnapshotList(Tree)` contract on SQLite and PostgreSQL: + pass, including the single read-only transaction and unchanged-state proof. +- Validation root: `/tmp/coldkeep-v11312-phase5.HwL0Yk/`, with unique storage + roots for each sequential suite and all Go caches beneath the root. + +PostgreSQL used the existing exact-name scratch harness through maintenance +database `postgres`. The first unrestricted attempt created and dropped +`coldkeep_bt_testcatalogcontractsnapshotgraphacrossback_26a0d3_1`, then stopped +because explicit scratch auto-bootstrap was not enabled. The complete rerun +created, bootstrapped, tested, and dropped +`coldkeep_bt_testcatalogcontractsnapshotgraphacrossback_26a25e_1`. Candidate +code and tracked files did not change between attempts. A final read-only query +against `pg_database` confirmed that neither exact name remained. The separate +production engine contract created and dropped +`coldkeep_bt_testenginesnapshotselectorsacrossbackends__26cb75_1`; the final +query also confirmed that exact name is absent. + +The backend harness now logs each exact scratch database name at creation and +drop, making this lifecycle auditable without prefix-wide cleanup. + +Phase 6 is authorized to implement ordered legacy and packed chunk placements +with parity against the current restore metadata queries. It may not adopt the +new restore plan in production. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index aab8d57b..7d9cd9c0 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–4 are Complete. Phase 5 is Next: implement deterministic snapshot -graph loading for SQLite and PostgreSQL and adopt the graph in lineage -planning without moving rendering into the catalog. +Phases 0–5 are Complete. Phase 6 is Next: implement deterministic legacy and +packed chunk placement loading for SQLite and PostgreSQL, including complete +restore metadata and parity with the existing restore queries. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 383b86d6..11ff5761 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -74,11 +74,20 @@ ## Phase 5 — Snapshot Graph Implementation -**Phase status:** Next +**Phase status:** Complete + +- [x] SQLite and PostgreSQL return the same graph nodes and relationships. +- [x] Node, root, and child ordering is deterministic across repeated reads. +- [x] Missing historical parents remain explicit and do not create edges. +- [x] Cycles and malformed graph metadata fail with typed invariant context. +- [x] `snapshot list --tree` carries its flag into the engine and consumes the + catalog graph while rendering remains in the CLI. +- [x] Exact PostgreSQL scratch database creation/drop is logged and no Phase 5 + database remains. ## Phase 6 — Placement Implementation -**Phase status:** Not started +**Phase status:** Next ## Phase 7 — Restore-Plan Implementation diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index ddc02bda..c83b545f 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -308,15 +308,59 @@ func assertIDSet(t *testing.T, got map[int64]struct{}, want ...int64) { } } -// CAT-006 preserves the deliberately deferred API boundary and proves those -// methods cannot return partial results or mutate the catalog. +// CAT-006 proves deterministic snapshot graph parity and non-mutation. +func TestCatalogContractSnapshotGraphAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + first, err := svc.LoadSnapshotGraph(context.Background()) + if err != nil { + t.Fatalf("LoadSnapshotGraph: %v", err) + } + second, err := svc.LoadSnapshotGraph(context.Background()) + if err != nil { + t.Fatalf("LoadSnapshotGraph repeated: %v", err) + } + if !reflect.DeepEqual(first, second) { + t.Fatalf("graph is not deterministic: first=%+v second=%+v", first, second) + } + wantOrder := []string{"snap-full", "snap-tie-a", "snap-tie-b", "snap-child", "snap-null-label"} + gotOrder := make([]string, len(first.Nodes)) + for i, node := range first.Nodes { + gotOrder[i] = node.Snapshot.ID + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Fatalf("node order: got %v want %v", gotOrder, wantOrder) + } + wantRoots := []string{"snap-full", "snap-tie-a", "snap-tie-b", "snap-null-label"} + if !reflect.DeepEqual(first.RootIDs, wantRoots) { + t.Fatalf("roots: got %v want %v", first.RootIDs, wantRoots) + } + if first.Nodes[0].ParentState != catalog.SnapshotParentNone || !reflect.DeepEqual(first.Nodes[0].ChildIDs, []string{"snap-child"}) { + t.Fatalf("root relation: %+v", first.Nodes[0]) + } + if first.Nodes[3].ParentState != catalog.SnapshotParentPresent || len(first.Nodes[3].ChildIDs) != 0 { + t.Fatalf("child relation: %+v", first.Nodes[3]) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if graph, err := svc.LoadSnapshotGraph(cancelled); graph != nil || !catalog.IsCode(err, catalog.ErrorCancelled) || !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled graph: graph=%+v err=%v", graph, err) + } + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("graph read mutated state: before=%+v after=%+v", before, after) + } + }) +} + +// CAT-007 preserves the remaining deliberately deferred API boundary and proves +// those methods cannot return partial results or mutate the catalog. func TestCatalogContractDeferredMethodsAcrossBackends(t *testing.T) { forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { seedCatalogFixture(t, backend.DB) svc := catalog.NewServiceFromSQL(backend.DB) before := catalogStateCounts(t, backend.DB) - graph, err := svc.LoadSnapshotGraph(context.Background()) - assertDeferred(t, "LoadSnapshotGraph", err, graph) placements, err := svc.LoadChunkPlacements(context.Background(), 1) assertDeferred(t, "LoadChunkPlacements", err, placements) restorePlan, err := svc.LoadRestorePlanMetadata(context.Background(), catalog.RestorePlanInput{FileID: 1}) diff --git a/internal/catalog/graph.go b/internal/catalog/graph.go index c1f89b97..d1ae6751 100644 --- a/internal/catalog/graph.go +++ b/internal/catalog/graph.go @@ -1,11 +1,167 @@ package catalog -import "context" +import ( + "context" + "database/sql" + "errors" + "fmt" + "sort" + "time" +) -// LoadSnapshotGraph implements SnapshotGraphCatalog. -// -// Phase 5 implements the backend-neutral graph contract defined in types.go. +// LoadSnapshotGraph implements SnapshotGraphCatalog for SQLite and PostgreSQL. +// It performs one ordered read and builds relationships entirely in memory. func (s *Service) LoadSnapshotGraph(ctx context.Context) (*SnapshotGraph, error) { - _ = ctx - return nil, ErrNotImplemented + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, graphCatalogError(err) + } + rows, err := s.db.QueryContext(ctx, ` +SELECT id, type, COALESCE(label, ''), COALESCE(parent_id, ''), created_at +FROM snapshot +ORDER BY created_at ASC, id ASC`) + if err != nil { + return nil, graphCatalogError(fmt.Errorf("query snapshot graph: %w", err)) + } + defer func() { _ = rows.Close() }() + + refs := make([]SnapshotRef, 0) + for rows.Next() { + var ref SnapshotRef + var createdAt any + if err := rows.Scan(&ref.ID, &ref.Type, &ref.Label, &ref.ParentID, &createdAt); err != nil { + return nil, graphCatalogError(fmt.Errorf("scan snapshot graph row: %w", err)) + } + parsed, err := catalogTimestamp(createdAt) + if err != nil { + return nil, NewError(ErrorInvariantViolation, "load snapshot graph", "valid_snapshot_timestamp", fmt.Sprintf("snapshot %q has invalid created_at metadata", ref.ID), err) + } + ref.CreatedAt = parsed + refs = append(refs, ref) + } + if err := rows.Err(); err != nil { + return nil, graphCatalogError(fmt.Errorf("iterate snapshot graph rows: %w", err)) + } + return buildSnapshotGraph(refs) +} + +func buildSnapshotGraph(refs []SnapshotRef) (*SnapshotGraph, error) { + ordered := append([]SnapshotRef(nil), refs...) + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].CreatedAt.Equal(ordered[j].CreatedAt) { + return ordered[i].ID < ordered[j].ID + } + return ordered[i].CreatedAt.Before(ordered[j].CreatedAt) + }) + graph := &SnapshotGraph{ + Nodes: make([]SnapshotGraphNode, len(ordered)), + RootIDs: make([]string, 0), + } + index := make(map[string]int, len(ordered)) + for i, ref := range ordered { + if ref.ID == "" { + return nil, NewError(ErrorInvariantViolation, "load snapshot graph", "unique_nonempty_snapshot_id", "snapshot graph contains an empty ID", nil) + } + if _, exists := index[ref.ID]; exists { + return nil, NewError(ErrorInvariantViolation, "load snapshot graph", "unique_nonempty_snapshot_id", fmt.Sprintf("snapshot graph contains duplicate ID %q", ref.ID), nil) + } + index[ref.ID] = i + graph.Nodes[i] = SnapshotGraphNode{Snapshot: ref, ChildIDs: make([]string, 0)} + } + + for i := range graph.Nodes { + node := &graph.Nodes[i] + parentID := node.Snapshot.ParentID + if parentID == "" { + node.ParentState = SnapshotParentNone + graph.RootIDs = append(graph.RootIDs, node.Snapshot.ID) + continue + } + parentIndex, exists := index[parentID] + if !exists { + node.ParentState = SnapshotParentMissing + continue + } + node.ParentState = SnapshotParentPresent + graph.Nodes[parentIndex].ChildIDs = append(graph.Nodes[parentIndex].ChildIDs, node.Snapshot.ID) + } + + if cycleID := snapshotGraphCycle(graph.Nodes, index); cycleID != "" { + return nil, NewError(ErrorInvariantViolation, "load snapshot graph", "acyclic_snapshot_graph", fmt.Sprintf("snapshot graph contains a parent cycle involving %q", cycleID), nil) + } + return graph, nil +} + +func snapshotGraphCycle(nodes []SnapshotGraphNode, index map[string]int) string { + const ( + unvisited uint8 = iota + visiting + visited + ) + state := make([]uint8, len(nodes)) + var visit func(int) string + visit = func(i int) string { + switch state[i] { + case visiting: + return nodes[i].Snapshot.ID + case visited: + return "" + } + state[i] = visiting + parentID := nodes[i].Snapshot.ParentID + if parentIndex, exists := index[parentID]; exists { + if cycleID := visit(parentIndex); cycleID != "" { + return cycleID + } + } + state[i] = visited + return "" + } + for i := range nodes { + if cycleID := visit(i); cycleID != "" { + return cycleID + } + } + return "" +} + +func catalogTimestamp(value any) (time.Time, error) { + switch typed := value.(type) { + case time.Time: + return typed.UTC(), nil + case string: + return parseCatalogTimestampString(typed) + case []byte: + return parseCatalogTimestampString(string(typed)) + default: + return time.Time{}, fmt.Errorf("unsupported timestamp type %T", value) + } +} + +func parseCatalogTimestampString(value string) (time.Time, error) { + for _, layout := range []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05", + } { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed.UTC(), nil + } + } + return time.Time{}, fmt.Errorf("invalid timestamp %q", value) +} + +func graphCatalogError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return NewError(ErrorCancelled, "load snapshot graph", "", "snapshot graph load cancelled", err) + } + if errors.Is(err, sql.ErrNoRows) { + return NewError(ErrorNotFound, "load snapshot graph", "", "snapshot graph row not found", err) + } + return NewError(ErrorOperationFailed, "load snapshot graph", "", "snapshot graph query failed", err) } diff --git a/internal/catalog/graph_test.go b/internal/catalog/graph_test.go new file mode 100644 index 00000000..875d9bdc --- /dev/null +++ b/internal/catalog/graph_test.go @@ -0,0 +1,72 @@ +package catalog + +import ( + "reflect" + "testing" + "time" +) + +func TestBuildSnapshotGraphOrdersRelationsWithoutMutatingInput(t *testing.T) { + base := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) + input := []SnapshotRef{ + {ID: "child-b", ParentID: "root", CreatedAt: base.Add(time.Hour)}, + {ID: "root", CreatedAt: base}, + {ID: "child-a", ParentID: "root", CreatedAt: base.Add(time.Hour)}, + } + original := append([]SnapshotRef(nil), input...) + graph, err := buildSnapshotGraph(input) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(input, original) { + t.Fatalf("input mutated: got %+v want %+v", input, original) + } + if got, want := graph.RootIDs, []string{"root"}; !reflect.DeepEqual(got, want) { + t.Fatalf("roots=%v want=%v", got, want) + } + if got, want := graph.Nodes[0].ChildIDs, []string{"child-a", "child-b"}; !reflect.DeepEqual(got, want) { + t.Fatalf("children=%v want=%v", got, want) + } +} + +func TestBuildSnapshotGraphClassifiesMissingParentWithoutInventingEdge(t *testing.T) { + graph, err := buildSnapshotGraph([]SnapshotRef{{ID: "orphan", ParentID: "historical-parent", CreatedAt: time.Now().UTC()}}) + if err != nil { + t.Fatal(err) + } + if len(graph.RootIDs) != 0 { + t.Fatalf("missing-parent row became a true root: %v", graph.RootIDs) + } + if len(graph.Nodes) != 1 || graph.Nodes[0].ParentState != SnapshotParentMissing || len(graph.Nodes[0].ChildIDs) != 0 { + t.Fatalf("missing-parent classification: %+v", graph) + } +} + +func TestBuildSnapshotGraphRejectsCyclesAndDuplicateIDs(t *testing.T) { + base := time.Now().UTC() + for name, refs := range map[string][]SnapshotRef{ + "self cycle": {{ID: "a", ParentID: "a", CreatedAt: base}}, + "two node cycle": {{ID: "a", ParentID: "b", CreatedAt: base}, {ID: "b", ParentID: "a", CreatedAt: base}}, + "duplicate": {{ID: "a", CreatedAt: base}, {ID: "a", CreatedAt: base}}, + } { + t.Run(name, func(t *testing.T) { + graph, err := buildSnapshotGraph(refs) + if graph != nil || !IsCode(err, ErrorInvariantViolation) { + t.Fatalf("graph=%+v err=%v", graph, err) + } + }) + } +} + +func TestCatalogTimestampPreservesBackendRepresentations(t *testing.T) { + want := time.Date(2026, 8, 19, 12, 34, 56, 123456000, time.UTC) + for _, input := range []any{want, want.Format(time.RFC3339Nano), []byte(want.Format(time.RFC3339Nano))} { + got, err := catalogTimestamp(input) + if err != nil || !got.Equal(want) { + t.Fatalf("input=%T(%v) got=%v err=%v", input, input, got, err) + } + } + if _, err := catalogTimestamp("not-a-time"); err == nil { + t.Fatal("expected malformed timestamp error") + } +} diff --git a/internal/catalog/service_test.go b/internal/catalog/service_test.go index 6cc2008a..5858dbd6 100644 --- a/internal/catalog/service_test.go +++ b/internal/catalog/service_test.go @@ -389,25 +389,14 @@ func TestServiceLoadReachabilityRoots(t *testing.T) { assertReachabilityRoots(t, roots) } -// TestServiceDeferredMethodsReturnErrNotImplemented verifies all skeleton -// methods return ErrNotImplemented and not nil or panic. +// TestServiceDeferredMethodsReturnErrNotImplemented verifies the remaining +// placement/restore/GC skeleton methods return ErrNotImplemented. func TestServiceDeferredMethodsReturnErrNotImplemented(t *testing.T) { dbconn := openTestDB(t) svc := catalog.NewServiceFromSQL(dbconn) ctx := context.Background() before := countCatalogLogicalFiles(t, dbconn) - graph, err := svc.LoadSnapshotGraph(ctx) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadSnapshotGraph: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadSnapshotGraph: want catalog.IsDeferred=true, got %v", err) - } - if graph != nil { - t.Errorf("LoadSnapshotGraph: want nil graph on deferred path, got %+v", graph) - } - placements, err := svc.LoadChunkPlacements(ctx, 1) if !errors.Is(err, catalog.ErrNotImplemented) { t.Errorf("LoadChunkPlacements: want ErrNotImplemented via errors.Is, got %v", err) diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index b0d029ff..711cb1ab 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -419,6 +419,29 @@ type SnapshotMeta struct { FileCount int } +// SnapshotParentState distinguishes a root, a resolved parent, and historical +// missing-parent metadata without inventing a relationship. +type SnapshotParentState string + +const ( + SnapshotParentNone SnapshotParentState = "none" + SnapshotParentPresent SnapshotParentState = "present" + SnapshotParentMissing SnapshotParentState = "missing" +) + +// SnapshotGraphNode is one renderer-neutral lineage node. +type SnapshotGraphNode struct { + Snapshot SnapshotMeta + ParentState SnapshotParentState + ChildIDs []string +} + +// SnapshotGraph is ordered by created_at ascending, then snapshot ID. +type SnapshotGraph struct { + Nodes []SnapshotGraphNode + RootIDs []string +} + // SnapshotCreateRequest is the frozen active v1.13.9 Engine snapshot-create // request surface. // @@ -446,8 +469,7 @@ type SnapshotCreateResult struct { ParentID string } -// SnapshotListRequest is the active but provisional request contract for -// Engine.SnapshotList. +// SnapshotListRequest is the active request contract for Engine.SnapshotList. type SnapshotListRequest struct { // Type filters by snapshot type; empty means all. Type SnapshotType @@ -459,25 +481,19 @@ type SnapshotListRequest struct { // Limit caps the number of results when greater than zero. Limit int // Tree requests lineage-tree ordering/visualization data. - // This provisional view-shaping flag does not prove engine ownership of - // lineage presentation semantics. Read-side ownership and shape decisions are - // deferred to early v2.0. + // Rendering remains the caller's responsibility. Tree bool } -// SnapshotListResult is the active but provisional result contract for -// Engine.SnapshotList. -// -// TreeMode and TreeLines are provisional view-shaping fields. They do not -// prove engine ownership of lineage presentation semantics; read-side -// ownership and shape decisions are deferred to early v2.0. +// SnapshotListResult is the active result contract for Engine.SnapshotList. type SnapshotListResult struct { Snapshots []SnapshotMeta Count int // TreeMode echoes whether tree data was requested. TreeMode bool - // TreeLines holds renderer-neutral lineage rows when TreeMode is set. - TreeLines []string + // Graph is populated only when TreeMode is true. It contains metadata and + // relationships, never rendered lines. + Graph *SnapshotGraph } // SnapshotFile is a renderer-neutral file entry within a snapshot. diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index 5880e535..75f3e790 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -208,7 +208,18 @@ func verifyLevelFromString(s string) (verify.VerifyLevel, error) { } func (e *DefaultEngine) SnapshotList(ctx context.Context, req SnapshotListRequest) (SnapshotListResult, error) { - svc := catalog.NewServiceFromSQL(e.config.DB) + var tx *sql.Tx + catalogDB := catalog.DB(e.config.DB) + if req.Tree { + var err error + tx, err = e.config.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return SnapshotListResult{}, TranslateError("snapshot_list", fmt.Errorf("begin tree snapshot: %w", err)) + } + defer func() { _ = tx.Rollback() }() + catalogDB = tx + } + svc := catalog.NewService(catalogDB) filter := catalog.SnapshotFilter{ Type: string(req.Type), LabelSubstring: req.Label, @@ -230,13 +241,68 @@ func (e *DefaultEngine) SnapshotList(ctx context.Context, req SnapshotListReques CreatedAt: ref.CreatedAt, } } + var resultGraph *SnapshotGraph + if req.Tree { + graph, err := svc.LoadSnapshotGraph(ctx) + if err != nil { + return SnapshotListResult{}, TranslateError("snapshot_list", err) + } + resultGraph, metas = projectSelectedSnapshotGraph(graph, refs) + if err := tx.Commit(); err != nil { + return SnapshotListResult{}, TranslateError("snapshot_list", fmt.Errorf("commit tree snapshot: %w", err)) + } + } return SnapshotListResult{ Snapshots: metas, Count: len(metas), TreeMode: req.Tree, + Graph: resultGraph, }, nil } +func projectSelectedSnapshotGraph(graph *catalog.SnapshotGraph, selected []catalog.SnapshotRef) (*SnapshotGraph, []SnapshotMeta) { + selectedIDs := make(map[string]struct{}, len(selected)) + for _, ref := range selected { + selectedIDs[ref.ID] = struct{}{} + } + result := &SnapshotGraph{Nodes: make([]SnapshotGraphNode, 0, len(selected)), RootIDs: make([]string, 0)} + for _, node := range graph.Nodes { + if _, ok := selectedIDs[node.Snapshot.ID]; !ok { + continue + } + children := make([]string, 0, len(node.ChildIDs)) + for _, childID := range node.ChildIDs { + if _, ok := selectedIDs[childID]; ok { + children = append(children, childID) + } + } + result.Nodes = append(result.Nodes, SnapshotGraphNode{ + Snapshot: SnapshotMeta{ + ID: node.Snapshot.ID, + Type: SnapshotType(node.Snapshot.Type), + Label: node.Snapshot.Label, + ParentID: node.Snapshot.ParentID, + CreatedAt: node.Snapshot.CreatedAt, + }, + ParentState: SnapshotParentState(node.ParentState), + ChildIDs: children, + }) + } + for _, rootID := range graph.RootIDs { + if _, ok := selectedIDs[rootID]; ok { + result.RootIDs = append(result.RootIDs, rootID) + } + } + + // SnapshotList historically returns newest first. Deriving this projection + // from the graph consumes catalog ordering while preserving that contract. + metas := make([]SnapshotMeta, len(result.Nodes)) + for i := range result.Nodes { + metas[len(result.Nodes)-1-i] = result.Nodes[i].Snapshot + } + return result, metas +} + func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowRequest) (SnapshotShowResult, error) { svc := catalog.NewServiceFromSQL(e.config.DB) ref, err := svc.FindSnapshot(ctx, req.SnapshotID) diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 5d2e1730..5b39a79b 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/invariants" ) @@ -98,7 +99,11 @@ func TranslateError(operation string, err error) error { case errors.Is(err, ErrNotImplemented): code = ErrorUnsupported default: - if value, ok := invariants.Code(err); ok { + var catalogErr *catalog.Error + if errors.As(err, &catalogErr) && catalogErr != nil { + code = engineCodeFromCatalog(catalogErr.Code) + invariantCode = catalogErr.Invariant + } else if value, ok := invariants.Code(err); ok { code = ErrorInvariantViolation invariantCode = value } @@ -106,6 +111,25 @@ func TranslateError(operation string, err error) error { return NewError(code, operation, err.Error(), invariantCode, err) } +func engineCodeFromCatalog(code catalog.ErrorCode) ErrorCode { + switch code { + case catalog.ErrorInvalidArgument: + return ErrorInvalidArgument + case catalog.ErrorNotFound: + return ErrorNotFound + case catalog.ErrorUnsupported: + return ErrorUnsupported + case catalog.ErrorInvariantViolation: + return ErrorInvariantViolation + case catalog.ErrorConflict: + return ErrorConflict + case catalog.ErrorCancelled: + return ErrorCancelled + default: + return ErrorOperationFailed + } +} + // TranslateErrorAs applies a caller-selected semantic classification. Context // cancellation and invariant errors still use their universal classifications. func TranslateErrorAs(operation string, code ErrorCode, err error) error { diff --git a/internal/engine/snapshot_engine_test.go b/internal/engine/snapshot_engine_test.go index 4fe46d1a..db12a357 100644 --- a/internal/engine/snapshot_engine_test.go +++ b/internal/engine/snapshot_engine_test.go @@ -3,6 +3,8 @@ package engine_test import ( "context" "database/sql" + "errors" + "reflect" "strings" "testing" "time" @@ -189,6 +191,68 @@ func TestSnapshotListLabelAndMeta(t *testing.T) { } } +func TestSnapshotListTreeConsumesCatalogGraphAndPreservesListOrder(t *testing.T) { + db := openSnapshotTestDB(t) + base := time.Date(2026, 8, 19, 8, 0, 0, 0, time.UTC) + insertTestSnapshot(t, db, "root", "full", "root", "", base) + insertTestSnapshot(t, db, "child-b", "full", "b", "root", base.Add(time.Hour)) + insertTestSnapshot(t, db, "child-a", "full", "a", "root", base.Add(time.Hour)) + + eng, err := engine.New(engine.Config{DB: db}) + if err != nil { + t.Fatal(err) + } + result, err := eng.SnapshotList(context.Background(), engine.SnapshotListRequest{Tree: true}) + if err != nil { + t.Fatalf("SnapshotList tree: %v", err) + } + if !result.TreeMode || result.Graph == nil { + t.Fatalf("missing tree graph: %+v", result) + } + if got, want := []string{result.Snapshots[0].ID, result.Snapshots[1].ID, result.Snapshots[2].ID}, []string{"child-b", "child-a", "root"}; !reflect.DeepEqual(got, want) { + t.Fatalf("list order got=%v want=%v", got, want) + } + if got, want := result.Graph.RootIDs, []string{"root"}; !reflect.DeepEqual(got, want) { + t.Fatalf("roots got=%v want=%v", got, want) + } + if got, want := result.Graph.Nodes[0].ChildIDs, []string{"child-a", "child-b"}; !reflect.DeepEqual(got, want) { + t.Fatalf("children got=%v want=%v", got, want) + } + if result.Graph.Nodes[1].ParentState != engine.SnapshotParentPresent { + t.Fatalf("parent state: %+v", result.Graph.Nodes[1]) + } + + plain, err := eng.SnapshotList(context.Background(), engine.SnapshotListRequest{}) + if err != nil { + t.Fatal(err) + } + if plain.TreeMode || plain.Graph != nil { + t.Fatalf("non-tree list exposed graph: %+v", plain) + } +} + +func TestSnapshotListTreeTranslatesCatalogGraphInvariant(t *testing.T) { + db := openSnapshotTestDB(t) + base := time.Date(2026, 8, 19, 8, 0, 0, 0, time.UTC) + insertTestSnapshot(t, db, "a", "full", "", "", base) + insertTestSnapshot(t, db, "b", "full", "", "a", base.Add(time.Hour)) + if _, err := db.Exec(`UPDATE snapshot SET parent_id = 'b' WHERE id = 'a'`); err != nil { + t.Fatal(err) + } + eng, err := engine.New(engine.Config{DB: db}) + if err != nil { + t.Fatal(err) + } + _, err = eng.SnapshotList(context.Background(), engine.SnapshotListRequest{Tree: true}) + if !engine.IsCode(err, engine.ErrorInvariantViolation) { + t.Fatalf("cycle error classification: %v", err) + } + var typed *engine.Error + if !errors.As(err, &typed) || typed.InvariantCode != "acyclic_snapshot_graph" { + t.Fatalf("cycle invariant context: %v", err) + } +} + // TestSnapshotShowNotFound verifies the engine returns an error for a missing snapshot. func TestSnapshotShowNotFound(t *testing.T) { db := openSnapshotTestDB(t) diff --git a/internal/engine/snapshot_selector_backend_contract_test.go b/internal/engine/snapshot_selector_backend_contract_test.go index 5f0a1d32..e5a80bc2 100644 --- a/internal/engine/snapshot_selector_backend_contract_test.go +++ b/internal/engine/snapshot_selector_backend_contract_test.go @@ -31,6 +31,16 @@ func TestEngineSnapshotSelectorsAcrossBackends(t *testing.T) { if err != nil || !reflect.DeepEqual(snapshotIDs(filtered), []string{"snap-target"}) { t.Fatalf("label SnapshotList: got (%+v, %v)", filtered, err) } + tree, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{Tree: true}) + if err != nil || tree.Graph == nil || !tree.TreeMode || !reflect.DeepEqual(snapshotIDs(tree), []string{"snap-target", "snap-base", "snap-root"}) { + t.Fatalf("tree SnapshotList: got (%+v, %v)", tree, err) + } + if got, want := tree.Graph.RootIDs, []string{"snap-root"}; !reflect.DeepEqual(got, want) { + t.Fatalf("tree roots: got %v want %v", got, want) + } + if got, want := tree.Graph.Nodes[0].ChildIDs, []string{"snap-base"}; !reflect.DeepEqual(got, want) { + t.Fatalf("tree root children: got %v want %v", got, want) + } query := engine.SnapshotQuery{ Path: "docs/added.txt", diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 396c8015..3fba1eb7 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -133,6 +133,9 @@ type SnapshotListFilter struct { Since *time.Time Until *time.Time Limit int + // Tree requests engine-owned graph planning. The snapshot-domain SQL list + // ignores this presentation-neutral routing flag. + Tree bool } type SnapshotFileEntry struct { diff --git a/internal/testutil/backendtest/backendtest.go b/internal/testutil/backendtest/backendtest.go index ff100d36..943c5385 100644 --- a/internal/testutil/backendtest/backendtest.go +++ b/internal/testutil/backendtest/backendtest.go @@ -116,6 +116,7 @@ func openPostgres(t *testing.T, schema SchemaMode) Backend { if _, err := admin.Exec("CREATE DATABASE " + quoteIdentifier(name)); err != nil { t.Fatalf("create PostgreSQL scratch database %q: %v", name, err) } + t.Logf("created PostgreSQL scratch database %q", name) var tested *sql.DB // This cleanup is registered after the admin close cleanup, so it runs first. @@ -124,6 +125,7 @@ func openPostgres(t *testing.T, schema SchemaMode) Backend { cleanupScratchDatabase(admin, tested, name, func(format string, args ...any) { t.Errorf(format, args...) }) + t.Logf("dropped PostgreSQL scratch database %q", name) }) tested = openPostgresConnection(t, name, "scratch database") ensureSchema(t, tested, schema, "PostgreSQL") From 75837ca3d8cf2a9df6690434f5f0ce8365404e2e Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 13:06:33 +0200 Subject: [PATCH 07/27] catalog: implement chunk placement loading --- docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...1.13.12-phase6-placement-implementation.md | 69 +++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 15 +- internal/catalog/backend_contract_test.go | 2 - internal/catalog/contracts.go | 33 ++- internal/catalog/placement.go | 244 +++++++++++++++++- .../placement_backend_contract_test.go | 185 +++++++++++++ internal/catalog/service_test.go | 13 +- 10 files changed, 546 insertions(+), 29 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase6-placement-implementation.md create mode 100644 internal/catalog/placement_backend_contract_test.go diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index ea2a0c3f..42bc5150 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–5 are complete; Phase 6 is next. + 0–6 are complete; Phase 7 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 7b9d5b05..f9b9fe4f 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -45,11 +45,13 @@ ## Phase 6 — Placement Implementation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 6 placements](v1.13.12-phase6-placement-implementation.md). ## Phase 7 — Restore-Plan Implementation -**Status:** Not started +**Status:** Next ## Phase 8 — Restore Production Adoption diff --git a/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md b/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md new file mode 100644 index 00000000..80d150ef --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md @@ -0,0 +1,69 @@ +# Coldkeep v1.13.12 Phase 6 — Placement Implementation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `6 — Placement Implementation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Implementation + +`catalog.Service.LoadChunkPlacements` now loads a complete ordered logical-file +recipe on SQLite and PostgreSQL. It validates the logical file first so a +missing file is distinguishable from a valid zero-length recipe, then performs +one ordered placement query. + +Each result contains chunk order, identity, hash, size, origin chunker version, +and lifecycle status plus exactly one placement variant: + +- Legacy placements carry the block descriptor, codec/version, plaintext and + stored sizes, nonce, container identity/state/sizes, and container offset. +- Packed placements carry the storage block ID, codec/version, compression + codec/level/sizes, logical/compressed/physical hashes, container + identity/state/sizes, container offset, and exact chunk segment. + +Coldkeep packed writes may retain a legacy companion `blocks` row. A valid +`chunk_block_refs` row is authoritative in that case; the catalog emits the +packed variant only. It falls back to the legacy variant only when no packed +reference exists. + +The implementation fails closed, with no partial recipe, on missing placement +metadata, non-contiguous ordering, invalid codec/nonce/compression contracts, +out-of-bounds container or segment ranges, missing mandatory block hashes, +integer overflow, or a recipe-size/logical-file-size mismatch. Quarantine is +preserved as catalog truth rather than erased; restore execution remains +responsible for its existing refusal behavior. + +## Compatibility and parity + +Shared tests compare the new recipe directly with the current production +restore metadata query. Legacy block fields, packed hash fields, chunk identity, +ordering, container identity, and maximum sizes match. Packed resolver fields +and storage-block metadata are asserted directly. No restore path consumes the +new catalog method in this phase; Phase 8 owns production adoption. + +No schema, migration, stored row, storage byte, transaction, lock, CLI/JSON +output, exit code, or backend default changed. + +## Validation + +- `go test -count=1 ./internal/catalog`: pass. +- `go test -race -count=1 ./internal/catalog`: pass. +- `go test -count=1 ./internal/engine`: pass. +- Shared mixed legacy/packed placement and restore-query parity contract on + SQLite and PostgreSQL: pass. +- Missing file, invalid ID, zero-length file, missing placement, cancellation, + deterministic repeat, no-mutation, tagged-union, bounds, and transform tests: + pass. +- Validation root: `/tmp/coldkeep-v11312-phase6.3BxWXO/`, with all Go caches + and unique suite storage roots below it. + +The initial PostgreSQL implementation pass created and dropped +`coldkeep_bt_testcatalogcontractchunkplacementsacrossba_26eace_1`. Contract +hardening then changed the candidate, so that result is not final-head proof. +The complete final-code rerun created, tested, and dropped +`coldkeep_bt_testcatalogcontractchunkplacementsacrossba_26ef59_1`. A read-only +maintenance query confirmed that neither exact database remains. + +Phase 7 is authorized to implement selector resolution and immutable restore +plan construction inside the caller's transaction. It may not yet replace the +production storage restore query or byte-I/O path. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 7d9cd9c0..0f748204 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–5 are Complete. Phase 6 is Next: implement deterministic legacy and -packed chunk placement loading for SQLite and PostgreSQL, including complete -restore metadata and parity with the existing restore queries. +Phases 0–6 are Complete. Phase 7 is Next: resolve file-ID, current stored-path, +and snapshot-path selectors and construct complete immutable restore plans +inside the caller's transaction. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 11ff5761..6c7ab9b1 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -87,11 +87,22 @@ ## Phase 6 — Placement Implementation -**Phase status:** Next +**Phase status:** Complete + +- [x] One ordered query represents legacy and packed placement metadata on both + SQLite and PostgreSQL. +- [x] Packed references take authority over compatibility companion rows and + exactly one tagged placement is returned per chunk. +- [x] Codec, nonce, compression, hash, segment, container, order, overflow, and + total recipe-size invariants fail closed with no partial result. +- [x] Zero-length, missing, malformed, cancelled, deterministic, and read-only + behaviors are directly tested. +- [x] New placement fields have parity with the existing restore metadata and + packed resolver queries. ## Phase 7 — Restore-Plan Implementation -**Phase status:** Not started +**Phase status:** Next ## Phase 8 — Restore Production Adoption diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index c83b545f..c3c4dc90 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -361,8 +361,6 @@ func TestCatalogContractDeferredMethodsAcrossBackends(t *testing.T) { seedCatalogFixture(t, backend.DB) svc := catalog.NewServiceFromSQL(backend.DB) before := catalogStateCounts(t, backend.DB) - placements, err := svc.LoadChunkPlacements(context.Background(), 1) - assertDeferred(t, "LoadChunkPlacements", err, placements) restorePlan, err := svc.LoadRestorePlanMetadata(context.Background(), catalog.RestorePlanInput{FileID: 1}) assertDeferred(t, "LoadRestorePlanMetadata", err, restorePlan) gcPlan, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{}) diff --git a/internal/catalog/contracts.go b/internal/catalog/contracts.go index b9a8b4bc..2ee6dc04 100644 --- a/internal/catalog/contracts.go +++ b/internal/catalog/contracts.go @@ -52,6 +52,9 @@ func ValidateChunkPlacement(placement ChunkPlacementRef) error { if placement.Legacy.BlockID <= 0 || placement.Legacy.Codec == "" || placement.Legacy.FormatVersion <= 0 || placement.Legacy.PlaintextSize <= 0 || placement.Legacy.StoredSize <= 0 || !validContainerRead(placement.Legacy.Container, placement.Legacy.ContainerOffset, placement.Legacy.StoredSize) { return invalidPlacement(placement, "legacy block and container bounds must be valid") } + if placement.Legacy.PlaintextSize != placement.ChunkSize || !validLegacyCodec(placement.Legacy.Codec, placement.Legacy.Nonce) { + return invalidPlacement(placement, "legacy codec, nonce, and plaintext size must match the chunk") + } case PlacementPacked: if placement.Packed == nil || placement.Legacy != nil { return invalidPlacement(placement, "packed placement must contain only packed metadata") @@ -59,14 +62,42 @@ func ValidateChunkPlacement(placement ChunkPlacementRef) error { if placement.Packed.BlockID <= 0 || placement.Packed.Codec == "" || placement.Packed.FormatVersion <= 0 || placement.Packed.PlaintextSize <= 0 || placement.Packed.CompressionCodec == "" || placement.Packed.StoredSize <= 0 || len(placement.Packed.BlockHash) == 0 || placement.Packed.OffsetInBlock < 0 || placement.Packed.SizeInBlock <= 0 || placement.Packed.OffsetInBlock+placement.Packed.SizeInBlock > placement.Packed.PlaintextSize || !validContainerRead(placement.Packed.Container, placement.Packed.ContainerOffset, placement.Packed.StoredSize) { return invalidPlacement(placement, "packed block, segment, and container bounds must be valid") } + if placement.Packed.SizeInBlock != placement.ChunkSize || !validPackedTransforms(placement.Packed) { + return invalidPlacement(placement, "packed codec, compression, and segment size must be valid") + } default: return invalidPlacement(placement, "placement kind must be legacy or packed") } return nil } +func validLegacyCodec(codec string, nonce []byte) bool { + switch codec { + case "plain": + return len(nonce) == 0 + case "aes-gcm": + return len(nonce) == 12 + default: + return false + } +} + +func validPackedTransforms(placement *PackedChunkPlacement) bool { + if placement == nil || (placement.Codec != "none" && placement.Codec != "aes-gcm") { + return false + } + switch placement.CompressionCodec { + case "none": + return placement.CompressionLevel == nil && (placement.CompressedSize == nil || *placement.CompressedSize > 0) + case "zstd": + return placement.CompressionLevel != nil && *placement.CompressionLevel >= 1 && *placement.CompressionLevel <= 9 && placement.CompressedSize != nil && *placement.CompressedSize > 0 + default: + return false + } +} + func validContainerRead(container ContainerPlacementRef, offset, size int64) bool { - if container.ID <= 0 || container.Filename == "" || container.Quarantined || offset < 0 || size <= 0 || container.CurrentSize < 0 || container.MaxSize <= 0 { + if container.ID <= 0 || container.Filename == "" || offset < 0 || size <= 0 || container.CurrentSize < 0 || container.MaxSize <= 0 { return false } end := offset + size diff --git a/internal/catalog/placement.go b/internal/catalog/placement.go index 7cd0e2c5..e58cb742 100644 --- a/internal/catalog/placement.go +++ b/internal/catalog/placement.go @@ -1,11 +1,243 @@ package catalog -import "context" +import ( + "context" + "database/sql" + "errors" + "fmt" +) -// LoadChunkPlacements implements PlacementCatalog. -// -// Phase 6 implements the strict packed/legacy union defined in types.go. +// LoadChunkPlacements returns one complete placement per logical recipe entry. +// Packed references are authoritative when a chunk also has a legacy companion +// row; otherwise the legacy block is returned. The method never returns a +// partial recipe. func (s *Service) LoadChunkPlacements(ctx context.Context, logicalFileID int64) ([]ChunkPlacementRef, error) { - _, _ = ctx, logicalFileID - return nil, ErrNotImplemented + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, placementCatalogError(err) + } + if logicalFileID <= 0 { + return nil, NewError(ErrorInvalidArgument, "load chunk placements", "positive_logical_file_id", "logical file ID must be positive", nil) + } + + var totalSize int64 + if err := s.db.QueryRowContext(ctx, `SELECT total_size FROM logical_file WHERE id = $1`, logicalFileID).Scan(&totalSize); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, NewError(ErrorNotFound, "load chunk placements", "", fmt.Sprintf("logical file %d not found", logicalFileID), err) + } + return nil, placementCatalogError(fmt.Errorf("query logical file %d: %w", logicalFileID, err)) + } + + rows, err := s.db.QueryContext(ctx, chunkPlacementQuery, logicalFileID) + if err != nil { + return nil, placementCatalogError(fmt.Errorf("query logical file %d placements: %w", logicalFileID, err)) + } + defer func() { _ = rows.Close() }() + + placements := make([]ChunkPlacementRef, 0) + var expectedOrder int64 + var recipeSize int64 + for rows.Next() { + scanned, err := scanChunkPlacement(rows) + if err != nil { + return nil, placementCatalogError(fmt.Errorf("scan logical file %d placement: %w", logicalFileID, err)) + } + placement, err := scanned.toPlacement() + if err != nil { + return nil, err + } + if placement.ChunkOrder != expectedOrder { + return nil, NewError(ErrorInvariantViolation, "load chunk placements", "contiguous_chunk_order", fmt.Sprintf("logical file %d has chunk order %d; expected %d", logicalFileID, placement.ChunkOrder, expectedOrder), nil) + } + if err := ValidateChunkPlacement(placement); err != nil { + return nil, err + } + placements = append(placements, placement) + if placement.ChunkSize > 0 && recipeSize > int64(^uint64(0)>>1)-placement.ChunkSize { + return nil, NewError(ErrorInvariantViolation, "load chunk placements", "recipe_size_no_overflow", fmt.Sprintf("logical file %d chunk sizes overflow int64", logicalFileID), nil) + } + recipeSize += placement.ChunkSize + expectedOrder++ + } + if err := rows.Err(); err != nil { + return nil, placementCatalogError(fmt.Errorf("iterate logical file %d placements: %w", logicalFileID, err)) + } + if len(placements) == 0 && totalSize != 0 { + return nil, NewError(ErrorInvariantViolation, "load chunk placements", "nonempty_file_has_recipe", fmt.Sprintf("logical file %d has size %d but no chunk recipe", logicalFileID, totalSize), nil) + } + if recipeSize != totalSize { + return nil, NewError(ErrorInvariantViolation, "load chunk placements", "recipe_size_matches_logical_file", fmt.Sprintf("logical file %d recipe size %d does not match total size %d", logicalFileID, recipeSize, totalSize), nil) + } + return placements, nil +} + +const chunkPlacementQuery = ` +SELECT + fc.chunk_order, + c.id, c.chunk_hash, c.size, c.chunker_version, c.status, + r.block_id, r.offset_in_block, r.size_in_block, + sb.format_version, sb.codec, sb.plaintext_size, + sb.compression_codec, sb.compression_level, sb.compressed_size, + sb.stored_size, sb.block_hash, sb.compressed_hash, sb.physical_hash, + sb.container_id, sb.container_offset, + pc.filename, pc.sealed, pc.sealing, pc.container_hash, pc.quarantine, + pc.current_size, pc.max_size, + b.id, b.codec, b.format_version, b.plaintext_size, b.stored_size, + b.nonce, b.container_id, b.block_offset, + lc.filename, lc.sealed, lc.sealing, lc.container_hash, lc.quarantine, + lc.current_size, lc.max_size +FROM file_chunk fc +JOIN chunk c ON c.id = fc.chunk_id +LEFT JOIN chunk_block_refs r ON r.chunk_id = c.id +LEFT JOIN storage_blocks sb ON sb.id = r.block_id +LEFT JOIN container pc ON pc.id = sb.container_id +LEFT JOIN blocks b ON b.chunk_id = c.id +LEFT JOIN container lc ON lc.id = b.container_id +WHERE fc.logical_file_id = $1 +ORDER BY fc.chunk_order ASC` + +type chunkPlacementScan struct { + chunkOrder, chunkID, chunkSize int64 + chunkHash, chunkerVersion, chunkStatus string + packedBlockID, packedOffset, packedSize sql.NullInt64 + packedFormat sql.NullInt64 + packedCodec, compressionCodec sql.NullString + packedPlaintext, compressionLevel, compressedSize sql.NullInt64 + packedStored sql.NullInt64 + blockHash, compressedHash, physicalHash []byte + packedContainerID, packedContainerOffset sql.NullInt64 + packedContainer nullableContainer + legacyBlockID, legacyFormat, legacyPlaintext sql.NullInt64 + legacyCodec sql.NullString + legacyStored sql.NullInt64 + legacyNonce []byte + legacyContainerID, legacyContainerOffset sql.NullInt64 + legacyContainer nullableContainer +} + +type nullableContainer struct { + filename, hash sql.NullString + sealed, sealing, quarantined sql.NullBool + currentSize, maxSize sql.NullInt64 +} + +func scanChunkPlacement(rows *sql.Rows) (chunkPlacementScan, error) { + var row chunkPlacementScan + err := rows.Scan( + &row.chunkOrder, + &row.chunkID, &row.chunkHash, &row.chunkSize, &row.chunkerVersion, &row.chunkStatus, + &row.packedBlockID, &row.packedOffset, &row.packedSize, + &row.packedFormat, &row.packedCodec, &row.packedPlaintext, + &row.compressionCodec, &row.compressionLevel, &row.compressedSize, + &row.packedStored, &row.blockHash, &row.compressedHash, &row.physicalHash, + &row.packedContainerID, &row.packedContainerOffset, + &row.packedContainer.filename, &row.packedContainer.sealed, &row.packedContainer.sealing, + &row.packedContainer.hash, &row.packedContainer.quarantined, + &row.packedContainer.currentSize, &row.packedContainer.maxSize, + &row.legacyBlockID, &row.legacyCodec, &row.legacyFormat, &row.legacyPlaintext, &row.legacyStored, + &row.legacyNonce, &row.legacyContainerID, &row.legacyContainerOffset, + &row.legacyContainer.filename, &row.legacyContainer.sealed, &row.legacyContainer.sealing, + &row.legacyContainer.hash, &row.legacyContainer.quarantined, + &row.legacyContainer.currentSize, &row.legacyContainer.maxSize, + ) + return row, err +} + +func (row chunkPlacementScan) toPlacement() (ChunkPlacementRef, error) { + placement := ChunkPlacementRef{ + ChunkOrder: row.chunkOrder, ChunkID: row.chunkID, ChunkHash: row.chunkHash, + ChunkSize: row.chunkSize, ChunkerVersion: row.chunkerVersion, ChunkStatus: row.chunkStatus, + } + if row.packedBlockID.Valid { + packed, err := row.packedPlacement() + if err != nil { + return ChunkPlacementRef{}, err + } + placement.Kind, placement.Packed = PlacementPacked, packed + return placement, nil + } + if row.legacyBlockID.Valid { + legacy, err := row.legacyPlacement() + if err != nil { + return ChunkPlacementRef{}, err + } + placement.Kind, placement.Legacy = PlacementLegacy, legacy + return placement, nil + } + return ChunkPlacementRef{}, invalidPlacement(placement, "chunk has neither packed nor legacy placement metadata") +} + +func (row chunkPlacementScan) packedPlacement() (*PackedChunkPlacement, error) { + if !row.packedOffset.Valid || !row.packedSize.Valid || !row.packedFormat.Valid || !row.packedCodec.Valid || !row.packedPlaintext.Valid || !row.compressionCodec.Valid || !row.packedStored.Valid || !row.packedContainerID.Valid || !row.packedContainerOffset.Valid { + return nil, invalidPlacement(ChunkPlacementRef{ChunkID: row.chunkID}, "packed placement metadata is incomplete") + } + container, err := row.packedContainer.value(row.packedContainerID.Int64) + if err != nil { + return nil, invalidPlacement(ChunkPlacementRef{ChunkID: row.chunkID}, err.Error()) + } + return &PackedChunkPlacement{ + BlockID: row.packedBlockID.Int64, FormatVersion: int(row.packedFormat.Int64), Codec: row.packedCodec.String, + PlaintextSize: row.packedPlaintext.Int64, CompressionCodec: row.compressionCodec.String, + CompressionLevel: optionalInt(row.compressionLevel), CompressedSize: optionalInt64(row.compressedSize), + StoredSize: row.packedStored.Int64, BlockHash: cloneBytes(row.blockHash), + CompressedHash: cloneBytes(row.compressedHash), PhysicalHash: cloneBytes(row.physicalHash), + Container: container, ContainerOffset: row.packedContainerOffset.Int64, + OffsetInBlock: row.packedOffset.Int64, SizeInBlock: row.packedSize.Int64, + }, nil +} + +func (row chunkPlacementScan) legacyPlacement() (*LegacyChunkPlacement, error) { + if !row.legacyCodec.Valid || !row.legacyFormat.Valid || !row.legacyPlaintext.Valid || !row.legacyStored.Valid || !row.legacyContainerID.Valid || !row.legacyContainerOffset.Valid { + return nil, invalidPlacement(ChunkPlacementRef{ChunkID: row.chunkID}, "legacy placement metadata is incomplete") + } + container, err := row.legacyContainer.value(row.legacyContainerID.Int64) + if err != nil { + return nil, invalidPlacement(ChunkPlacementRef{ChunkID: row.chunkID}, err.Error()) + } + return &LegacyChunkPlacement{ + BlockID: row.legacyBlockID.Int64, Codec: row.legacyCodec.String, FormatVersion: int(row.legacyFormat.Int64), + PlaintextSize: row.legacyPlaintext.Int64, StoredSize: row.legacyStored.Int64, Nonce: cloneBytes(row.legacyNonce), + Container: container, ContainerOffset: row.legacyContainerOffset.Int64, + }, nil +} + +func (container nullableContainer) value(id int64) (ContainerPlacementRef, error) { + if !container.filename.Valid || !container.sealed.Valid || !container.sealing.Valid || !container.quarantined.Valid || !container.currentSize.Valid || !container.maxSize.Valid { + return ContainerPlacementRef{}, fmt.Errorf("container %d metadata is incomplete", id) + } + return ContainerPlacementRef{ + ID: id, Filename: container.filename.String, Sealed: container.sealed.Bool, + Sealing: container.sealing.Bool, ContainerHash: container.hash.String, + Quarantined: container.quarantined.Bool, CurrentSize: container.currentSize.Int64, + MaxSize: container.maxSize.Int64, + }, nil +} + +func optionalInt(value sql.NullInt64) *int { + if !value.Valid { + return nil + } + converted := int(value.Int64) + return &converted +} + +func optionalInt64(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + converted := value.Int64 + return &converted +} + +func cloneBytes(value []byte) []byte { + return append([]byte(nil), value...) +} + +func placementCatalogError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return NewError(ErrorCancelled, "load chunk placements", "", "chunk placement load cancelled", err) + } + return NewError(ErrorOperationFailed, "load chunk placements", "", "chunk placement query failed", err) } diff --git a/internal/catalog/placement_backend_contract_test.go b/internal/catalog/placement_backend_contract_test.go new file mode 100644 index 00000000..a6c80266 --- /dev/null +++ b/internal/catalog/placement_backend_contract_test.go @@ -0,0 +1,185 @@ +package catalog_test + +import ( + "bytes" + "context" + "errors" + "reflect" + "testing" + + "github.com/franchoy/coldkeep/internal/catalog" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +const ( + placementLogicalID int64 = 51 + placementEmptyID int64 = 52 + placementMalformedID int64 = 53 +) + +func TestCatalogContractChunkPlacementsAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + seedPlacementContractFixture(t, backend) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + + first, err := svc.LoadChunkPlacements(context.Background(), placementLogicalID) + if err != nil { + t.Fatalf("LoadChunkPlacements: %v", err) + } + second, err := svc.LoadChunkPlacements(context.Background(), placementLogicalID) + if err != nil { + t.Fatalf("LoadChunkPlacements repeated: %v", err) + } + if !reflect.DeepEqual(first, second) { + t.Fatalf("placements not deterministic: first=%+v second=%+v", first, second) + } + assertPlacementContractRows(t, first) + assertPlacementParityWithRestoreQueries(t, backend, first) + + empty, err := svc.LoadChunkPlacements(context.Background(), placementEmptyID) + if err != nil || len(empty) != 0 || empty == nil { + t.Fatalf("zero-length placements: got=%#v err=%v", empty, err) + } + if got, err := svc.LoadChunkPlacements(context.Background(), 999999); got != nil || !catalog.IsCode(err, catalog.ErrorNotFound) { + t.Fatalf("missing logical file: got=%+v err=%v", got, err) + } + if got, err := svc.LoadChunkPlacements(context.Background(), 0); got != nil || !catalog.IsCode(err, catalog.ErrorInvalidArgument) { + t.Fatalf("invalid ID: got=%+v err=%v", got, err) + } + if got, err := svc.LoadChunkPlacements(context.Background(), placementMalformedID); got != nil || !catalog.IsCode(err, catalog.ErrorInvariantViolation) { + t.Fatalf("missing placement: got=%+v err=%v", got, err) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if got, err := svc.LoadChunkPlacements(cancelled, placementLogicalID); got != nil || !catalog.IsCode(err, catalog.ErrorCancelled) || !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled placements: got=%+v err=%v", got, err) + } + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("placement reads mutated catalog: before=%+v after=%+v", before, after) + } + }) +} + +func seedPlacementContractFixture(t *testing.T, backend backendtest.Backend) { + t.Helper() + exec := func(query string, args ...any) { + t.Helper() + if _, err := backend.DB.ExecContext(context.Background(), query, args...); err != nil { + t.Fatalf("seed placement fixture: %v\nquery: %s", err, query) + } + } + for _, row := range []struct { + id, size int64 + name, hash string + }{ + {placementLogicalID, 20, "mixed.bin", "file-hash-mixed"}, + {placementEmptyID, 0, "empty.bin", "file-hash-empty"}, + {placementMalformedID, 1, "malformed.bin", "file-hash-malformed"}, + } { + exec(`INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, chunker_version, status) VALUES ($1,$2,$3,$4,1,'v2-fastcdc','COMPLETED')`, row.id, row.name, row.size, row.hash) + } + for _, row := range []struct { + id int64 + filename string + hash any + }{ + {301, "legacy.ckc", "legacy-container-hash"}, + {302, "packed.ckc", nil}, + } { + exec(`INSERT INTO container (id, filename, sealed, sealing, container_hash, quarantine, current_size, max_size) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, row.id, row.filename, true, false, row.hash, false, int64(500), int64(4096)) + } + for _, row := range []struct { + id, size int64 + hash string + }{ + {101, 9, "chunk-hash-legacy"}, + {102, 11, "chunk-hash-packed"}, + {103, 1, "chunk-hash-malformed"}, + } { + exec(`INSERT INTO chunk (id, chunk_hash, size, status, chunker_version) VALUES ($1,$2,$3,'COMPLETED','v2-fastcdc')`, row.id, row.hash, row.size) + } + exec(`INSERT INTO file_chunk (logical_file_id, chunk_id, chunk_order) VALUES ($1,101,0)`, placementLogicalID) + exec(`INSERT INTO file_chunk (logical_file_id, chunk_id, chunk_order) VALUES ($1,102,1)`, placementLogicalID) + exec(`INSERT INTO file_chunk (logical_file_id, chunk_id, chunk_order) VALUES ($1,103,0)`, placementMalformedID) + exec(`INSERT INTO blocks (id, chunk_id, codec, format_version, plaintext_size, stored_size, nonce, container_id, block_offset) VALUES (201,101,'plain',1,9,9,NULL,301,64)`) + // Packed writes retain this companion legacy row. The catalog must choose the + // packed reference below and return one placement form, not both. + exec(`INSERT INTO blocks (id, chunk_id, codec, format_version, plaintext_size, stored_size, nonce, container_id, block_offset) VALUES (202,102,'plain',1,11,11,NULL,302,120)`) + exec(`INSERT INTO storage_blocks (id, format_version, codec, plaintext_size, compression_codec, compression_level, compressed_size, stored_size, container_id, container_offset, block_hash, compressed_hash, physical_hash) VALUES (401,1,'none',100,'none',NULL,NULL,80,302,80,$1,$2,$3)`, []byte{1, 2, 3}, []byte{4, 5}, []byte{6, 7}) + exec(`INSERT INTO chunk_block_refs (chunk_id, block_id, offset_in_block, size_in_block) VALUES (102,401,20,11)`) +} + +func assertPlacementContractRows(t *testing.T, got []catalog.ChunkPlacementRef) { + t.Helper() + if len(got) != 2 { + t.Fatalf("placements len=%d got=%+v", len(got), got) + } + legacy := got[0] + if legacy.ChunkOrder != 0 || legacy.ChunkID != 101 || legacy.Kind != catalog.PlacementLegacy || legacy.Legacy == nil || legacy.Packed != nil { + t.Fatalf("legacy placement: %+v", legacy) + } + if legacy.Legacy.BlockID != 201 || legacy.Legacy.Container.ID != 301 || legacy.Legacy.ContainerOffset != 64 || legacy.Legacy.Container.Filename != "legacy.ckc" || legacy.Legacy.Container.ContainerHash != "legacy-container-hash" { + t.Fatalf("legacy metadata: %+v", legacy.Legacy) + } + packed := got[1] + if packed.ChunkOrder != 1 || packed.ChunkID != 102 || packed.Kind != catalog.PlacementPacked || packed.Packed == nil || packed.Legacy != nil { + t.Fatalf("packed placement: %+v", packed) + } + if packed.Packed.BlockID != 401 || packed.Packed.Container.ID != 302 || packed.Packed.ContainerOffset != 80 || packed.Packed.OffsetInBlock != 20 || packed.Packed.SizeInBlock != 11 { + t.Fatalf("packed metadata: %+v", packed.Packed) + } + if !bytes.Equal(packed.Packed.BlockHash, []byte{1, 2, 3}) || !bytes.Equal(packed.Packed.CompressedHash, []byte{4, 5}) || !bytes.Equal(packed.Packed.PhysicalHash, []byte{6, 7}) { + t.Fatalf("packed hashes: %+v", packed.Packed) + } +} + +func assertPlacementParityWithRestoreQueries(t *testing.T, backend backendtest.Backend, got []catalog.ChunkPlacementRef) { + t.Helper() + rows, err := backend.DB.QueryContext(context.Background(), ` +SELECT fc.chunk_order, c.id, c.chunk_hash, c.size, c.chunker_version, c.status, + COALESCE(b.block_offset,0), COALESCE(b.plaintext_size,c.size), + COALESCE(b.stored_size,c.size), COALESCE(b.codec,'plain'), + COALESCE(b.format_version,1), b.nonce, COALESCE(b.container_id,0), + ctr.filename, ctr.max_size, sb.block_hash, sb.compressed_hash, sb.physical_hash +FROM file_chunk fc +JOIN chunk c ON c.id=fc.chunk_id +LEFT JOIN blocks b ON b.chunk_id=c.id +LEFT JOIN chunk_block_refs r ON r.chunk_id=c.id +LEFT JOIN storage_blocks sb ON sb.id=r.block_id +LEFT JOIN container ctr ON ctr.id=COALESCE(b.container_id,sb.container_id) +WHERE fc.logical_file_id=$1 AND c.status='COMPLETED' +ORDER BY fc.chunk_order`, placementLogicalID) + if err != nil { + t.Fatal(err) + } + defer func() { _ = rows.Close() }() + index := 0 + for rows.Next() { + var order, chunkID, chunkSize, offset, plaintext, stored, format, containerID, maxSize int64 + var chunkHash, version, status, codec, filename string + var nonce, blockHash, compressedHash, physicalHash []byte + if err := rows.Scan(&order, &chunkID, &chunkHash, &chunkSize, &version, &status, &offset, &plaintext, &stored, &codec, &format, &nonce, &containerID, &filename, &maxSize, &blockHash, &compressedHash, &physicalHash); err != nil { + t.Fatal(err) + } + placement := got[index] + if placement.ChunkOrder != order || placement.ChunkID != chunkID || placement.ChunkHash != chunkHash || placement.ChunkSize != chunkSize || placement.ChunkerVersion != version || placement.ChunkStatus != status { + t.Fatalf("restore recipe parity row=%d placement=%+v", index, placement) + } + if placement.Kind == catalog.PlacementLegacy { + if placement.Legacy.ContainerOffset != offset || placement.Legacy.PlaintextSize != plaintext || placement.Legacy.StoredSize != stored || placement.Legacy.Codec != codec || int64(placement.Legacy.FormatVersion) != format || placement.Legacy.Container.ID != containerID || placement.Legacy.Container.Filename != filename || placement.Legacy.Container.MaxSize != maxSize || !bytes.Equal(placement.Legacy.Nonce, nonce) { + t.Fatalf("legacy restore parity: %+v", placement.Legacy) + } + } else if !bytes.Equal(placement.Packed.BlockHash, blockHash) || !bytes.Equal(placement.Packed.CompressedHash, compressedHash) || !bytes.Equal(placement.Packed.PhysicalHash, physicalHash) { + t.Fatalf("packed restore hash parity: %+v", placement.Packed) + } + index++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if index != len(got) { + t.Fatalf("restore parity rows=%d placements=%d", index, len(got)) + } +} diff --git a/internal/catalog/service_test.go b/internal/catalog/service_test.go index 5858dbd6..80c4ce24 100644 --- a/internal/catalog/service_test.go +++ b/internal/catalog/service_test.go @@ -390,24 +390,13 @@ func TestServiceLoadReachabilityRoots(t *testing.T) { } // TestServiceDeferredMethodsReturnErrNotImplemented verifies the remaining -// placement/restore/GC skeleton methods return ErrNotImplemented. +// restore/GC skeleton methods return ErrNotImplemented. func TestServiceDeferredMethodsReturnErrNotImplemented(t *testing.T) { dbconn := openTestDB(t) svc := catalog.NewServiceFromSQL(dbconn) ctx := context.Background() before := countCatalogLogicalFiles(t, dbconn) - placements, err := svc.LoadChunkPlacements(ctx, 1) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadChunkPlacements: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadChunkPlacements: want catalog.IsDeferred=true, got %v", err) - } - if placements != nil { - t.Errorf("LoadChunkPlacements: want nil placements on deferred path, got %+v", placements) - } - restorePlan, err := svc.LoadRestorePlanMetadata(ctx, catalog.RestorePlanInput{FileID: 1}) if !errors.Is(err, catalog.ErrNotImplemented) { t.Errorf("LoadRestorePlanMetadata: want ErrNotImplemented via errors.Is, got %v", err) From 536f5f1d373ed09ed557c61b1201e0417e8117ab Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 13:11:49 +0200 Subject: [PATCH 08/27] catalog: implement restore plan metadata --- docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...3.12-phase7-restore-plan-implementation.md | 64 +++++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 13 +- internal/catalog/backend_contract_test.go | 2 - .../placement_backend_contract_test.go | 100 +++++++++++++ internal/catalog/restore_plan.go | 134 +++++++++++++++++- internal/catalog/service_test.go | 15 +- internal/catalog/types.go | 1 + 10 files changed, 314 insertions(+), 29 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase7-restore-plan-implementation.md diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 42bc5150..c63d95f4 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–6 are complete; Phase 7 is next. + 0–7 are complete; Phase 8 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index f9b9fe4f..1133e30d 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -51,11 +51,13 @@ ## Phase 7 — Restore-Plan Implementation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 7 restore plans](v1.13.12-phase7-restore-plan-implementation.md). ## Phase 8 — Restore Production Adoption -**Status:** Not started +**Status:** Next ## Phase 9 — GC-Plan Implementation and Adoption diff --git a/docs/release/v1.13/v1.13.12-phase7-restore-plan-implementation.md b/docs/release/v1.13/v1.13.12-phase7-restore-plan-implementation.md new file mode 100644 index 00000000..5a0111e5 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase7-restore-plan-implementation.md @@ -0,0 +1,64 @@ +# Coldkeep v1.13.12 Phase 7 — Restore-Plan Implementation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `7 — Restore-Plan Implementation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Implementation + +`catalog.Service.LoadRestorePlanMetadata` now resolves exactly one typed +selector: + +- logical file ID; +- current stored path; +- snapshot ID plus snapshot path. + +Selector validation runs before any query. Current-path resolution preserves +nullable mode, mtime, UID, and GID plus metadata-completeness state. Snapshot +resolution preserves nullable size, mode, and mtime. The plan then loads the +complete logical-file identity/provenance and the Phase 6 ordered placement +recipe. + +The catalog never begins, commits, or rolls back a transaction. A service +constructed with a caller-owned transaction resolves the selector, logical +file, and all placements within that transaction. The dual-backend suite proves +transaction-only data is visible to the plan and absent after caller rollback. + +Missing selector targets return `not_found`; malformed selector unions return +`invalid_argument`; incomplete or inconsistent recipes return +`invariant_violation`; non-completed logical files return `conflict`; and +cancellation remains detectable through the typed catalog boundary. No error +returns a partial plan. + +## Compatibility boundary + +The complete plan supports legacy, packed, mixed-layout, and zero-length +logical files. No production restore execution consumes it yet. The existing +pinning, byte reads, transaction lifetime, integrity checks, destination +validation, atomic output, output text/JSON, and exit behavior remain unchanged +until the separately reviewed Phase 8 adoption. + +No schema, migration, persisted metadata, storage byte, lock, backend default, +or public contract changed. + +## Validation + +- `go test -count=1 ./internal/catalog`: pass. +- `go test -race -count=1 ./internal/catalog`: pass. +- `go test -count=1 ./internal/engine`: pass after the contract expansion. +- Shared selector/plan contract on SQLite and PostgreSQL: pass. +- All selectors, exact nullable metadata, identical recipe projection, + zero-length, mixed placement, invalid/missing/malformed/non-completed cases, + cancellation, transaction visibility/rollback, and unchanged-state checks: + pass. +- Validation root: `/tmp/coldkeep-v11312-phase7.lqEbfW/`, with isolated caches + and unique suite storage roots. + +PostgreSQL created, tested, and dropped exactly +`coldkeep_bt_testcatalogcontractrestoreplansacrossbacke_272451_1` through the +maintenance connection. A final read-only query confirmed it is absent. + +Phase 8 is authorized to replace the duplicated production restore metadata +query with this catalog plan while preserving pinning, transaction lifetime, +byte I/O, integrity verification, and atomic destination behavior exactly. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 0f748204..1f5dee6d 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–6 are Complete. Phase 7 is Next: resolve file-ID, current stored-path, -and snapshot-path selectors and construct complete immutable restore plans -inside the caller's transaction. +Phases 0–7 are Complete. Phase 8 is Next: adopt the catalog restore plan in the +production storage path while preserving pinning, transaction lifetime, +integrity checks, byte I/O, and atomic destination behavior. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 6c7ab9b1..c8f6e2b5 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -102,11 +102,20 @@ ## Phase 7 — Restore-Plan Implementation -**Phase status:** Next +**Phase status:** Complete + +- [x] File-ID, current stored-path, and snapshot-path selectors are exclusive + and resolved on both backends. +- [x] Current and snapshot source metadata preserves exact nullable fields. +- [x] Logical identity/provenance and ordered mixed-layout placements form one + complete plan. +- [x] Missing, malformed, non-completed, zero-length, and cancellation behavior + is typed and directly tested. +- [x] Caller-owned transaction visibility and rollback are proven. ## Phase 8 — Restore Production Adoption -**Phase status:** Not started +**Phase status:** Next ## Phase 9 — GC-Plan Implementation and Adoption diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index c3c4dc90..e82c7f29 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -361,8 +361,6 @@ func TestCatalogContractDeferredMethodsAcrossBackends(t *testing.T) { seedCatalogFixture(t, backend.DB) svc := catalog.NewServiceFromSQL(backend.DB) before := catalogStateCounts(t, backend.DB) - restorePlan, err := svc.LoadRestorePlanMetadata(context.Background(), catalog.RestorePlanInput{FileID: 1}) - assertDeferred(t, "LoadRestorePlanMetadata", err, restorePlan) gcPlan, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{}) assertDeferred(t, "LoadGCPlanMetadata", err, gcPlan) if after := catalogStateCounts(t, backend.DB); after != before { diff --git a/internal/catalog/placement_backend_contract_test.go b/internal/catalog/placement_backend_contract_test.go index a6c80266..44b9fbed 100644 --- a/internal/catalog/placement_backend_contract_test.go +++ b/internal/catalog/placement_backend_contract_test.go @@ -6,6 +6,7 @@ import ( "errors" "reflect" "testing" + "time" "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/testutil/backendtest" @@ -62,6 +63,105 @@ func TestCatalogContractChunkPlacementsAcrossBackends(t *testing.T) { }) } +func TestCatalogContractRestorePlansAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + seedPlacementContractFixture(t, backend) + mtime := time.Date(2026, 8, 19, 16, 30, 0, 0, time.UTC) + if _, err := backend.DB.ExecContext(context.Background(), `INSERT INTO physical_file (path,logical_file_id,mode,mtime,uid,gid,is_metadata_complete) VALUES ('current/mixed.bin',$1,420,$2,1000,1001,$3)`, placementLogicalID, mtime, true); err != nil { + t.Fatal(err) + } + if _, err := backend.DB.ExecContext(context.Background(), `INSERT INTO snapshot (id,created_at,type,label) VALUES ('restore-snap',$1,'full','restore')`, mtime); err != nil { + t.Fatal(err) + } + if _, err := backend.DB.ExecContext(context.Background(), `INSERT INTO snapshot_path (id,path) VALUES (701,'archive/mixed.bin')`); err != nil { + t.Fatal(err) + } + if _, err := backend.DB.ExecContext(context.Background(), `INSERT INTO snapshot_file (snapshot_id,path_id,logical_file_id,size,mode,mtime) VALUES ('restore-snap',701,$1,20,384,$2)`, placementLogicalID, mtime); err != nil { + t.Fatal(err) + } + if _, err := backend.DB.ExecContext(context.Background(), `INSERT INTO logical_file (id,original_name,total_size,file_hash,ref_count,chunker_version,status) VALUES (54,'processing.bin',0,'processing-hash',1,'v2-fastcdc','PROCESSING')`); err != nil { + t.Fatal(err) + } + + before := catalogStateCounts(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + byID := requireRestorePlan(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: placementLogicalID}) + if byID.LogicalFile.ID != placementLogicalID || byID.Source != (catalog.RestoreSourceRef{}) || len(byID.Placements) != 2 { + t.Fatalf("file-ID plan: %+v", byID) + } + stored := requireRestorePlan(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByStoredPath, StoredPath: "current/mixed.bin"}) + if stored.Source.StoredPath != "current/mixed.bin" || stored.Source.Mode == nil || *stored.Source.Mode != 420 || stored.Source.UID == nil || *stored.Source.UID != 1000 || stored.Source.GID == nil || *stored.Source.GID != 1001 || stored.Source.MTime == nil || !stored.Source.MTime.Equal(mtime) || !stored.Source.IsMetadataComplete { + t.Fatalf("stored-path source: %+v", stored.Source) + } + snapshotPlan := requireRestorePlan(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreBySnapshotPath, SnapshotID: "restore-snap", SnapshotPath: "archive/mixed.bin"}) + if snapshotPlan.Source.SnapshotID != "restore-snap" || snapshotPlan.Source.SnapshotPath != "archive/mixed.bin" || snapshotPlan.Source.Size == nil || *snapshotPlan.Source.Size != 20 || snapshotPlan.Source.Mode == nil || *snapshotPlan.Source.Mode != 384 || snapshotPlan.Source.MTime == nil || !snapshotPlan.Source.MTime.Equal(mtime) || !snapshotPlan.Source.IsMetadataComplete { + t.Fatalf("snapshot source: %+v", snapshotPlan.Source) + } + if !reflect.DeepEqual(byID.Placements, stored.Placements) || !reflect.DeepEqual(byID.Placements, snapshotPlan.Placements) { + t.Fatal("selectors produced different logical recipes") + } + empty := requireRestorePlan(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: placementEmptyID}) + if empty.LogicalFile.TotalSize != 0 || len(empty.Placements) != 0 || empty.Placements == nil { + t.Fatalf("empty plan: %+v", empty) + } + + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{}, catalog.ErrorInvalidArgument) + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: 999999}, catalog.ErrorNotFound) + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByStoredPath, StoredPath: "missing"}, catalog.ErrorNotFound) + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreBySnapshotPath, SnapshotID: "restore-snap", SnapshotPath: "missing"}, catalog.ErrorNotFound) + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: placementMalformedID}, catalog.ErrorInvariantViolation) + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: 54}, catalog.ErrorConflict) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if plan, err := svc.LoadRestorePlanMetadata(cancelled, catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: placementLogicalID}); plan != nil || !catalog.IsCode(err, catalog.ErrorCancelled) || !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled plan=%+v err=%v", plan, err) + } + + tx, err := backend.DB.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`INSERT INTO physical_file (path,logical_file_id,is_metadata_complete) VALUES ('transaction-only.bin',$1,$2)`, placementLogicalID, false); err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + txPlan, err := catalog.NewService(tx).LoadRestorePlanMetadata(context.Background(), catalog.RestorePlanInput{Selector: catalog.RestoreByStoredPath, StoredPath: "transaction-only.bin"}) + if err != nil || txPlan.Source.StoredPath != "transaction-only.bin" { + _ = tx.Rollback() + t.Fatalf("transaction plan=%+v err=%v", txPlan, err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + assertRestorePlanCode(t, svc, catalog.RestorePlanInput{Selector: catalog.RestoreByStoredPath, StoredPath: "transaction-only.bin"}, catalog.ErrorNotFound) + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("restore plan reads mutated catalog: before=%+v after=%+v", before, after) + } + }) +} + +func requireRestorePlan(t *testing.T, svc interface { + LoadRestorePlanMetadata(context.Context, catalog.RestorePlanInput) (*catalog.RestorePlanMetadata, error) +}, input catalog.RestorePlanInput) *catalog.RestorePlanMetadata { + t.Helper() + plan, err := svc.LoadRestorePlanMetadata(context.Background(), input) + if err != nil || plan == nil { + t.Fatalf("LoadRestorePlanMetadata(%+v): plan=%+v err=%v", input, plan, err) + } + return plan +} + +func assertRestorePlanCode(t *testing.T, svc interface { + LoadRestorePlanMetadata(context.Context, catalog.RestorePlanInput) (*catalog.RestorePlanMetadata, error) +}, input catalog.RestorePlanInput, code catalog.ErrorCode) { + t.Helper() + plan, err := svc.LoadRestorePlanMetadata(context.Background(), input) + if plan != nil || !catalog.IsCode(err, code) { + t.Fatalf("LoadRestorePlanMetadata(%+v): plan=%+v err=%v wantCode=%s", input, plan, err, code) + } +} + func seedPlacementContractFixture(t *testing.T, backend backendtest.Backend) { t.Helper() exec := func(query string, args ...any) { diff --git a/internal/catalog/restore_plan.go b/internal/catalog/restore_plan.go index e30d5c1d..257ee42e 100644 --- a/internal/catalog/restore_plan.go +++ b/internal/catalog/restore_plan.go @@ -1,11 +1,133 @@ package catalog -import "context" +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) -// LoadRestorePlanMetadata implements RestorePlanCatalog. -// -// Phase 7 implements selector resolution and immutable recipe construction. +// LoadRestorePlanMetadata resolves one selector and constructs a complete +// immutable recipe using the Service's injected DB or caller-owned transaction. func (s *Service) LoadRestorePlanMetadata(ctx context.Context, input RestorePlanInput) (*RestorePlanMetadata, error) { - _, _ = ctx, input - return nil, ErrNotImplemented + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, restoreCatalogError(err) + } + if err := ValidateRestorePlanInput(input); err != nil { + return nil, err + } + + logicalID, source, err := s.resolveRestoreSelector(ctx, input) + if err != nil { + return nil, err + } + logical, err := s.loadRestoreLogicalFile(ctx, logicalID) + if err != nil { + return nil, err + } + placements, err := s.LoadChunkPlacements(ctx, logicalID) + if err != nil { + return nil, err + } + if logical.Status != "COMPLETED" { + return nil, NewError(ErrorConflict, "load restore plan", "completed_logical_file", fmt.Sprintf("logical file %d is not completed", logicalID), nil) + } + if (logical.TotalSize == 0 && len(placements) != 0) || (logical.TotalSize > 0 && len(placements) == 0) { + return nil, NewError(ErrorInvariantViolation, "load restore plan", "recipe_matches_logical_file_size", fmt.Sprintf("logical file %d has inconsistent zero-length recipe metadata", logicalID), nil) + } + return &RestorePlanMetadata{Selector: cloneRestoreSelector(input), LogicalFile: logical, Source: source, Placements: placements}, nil +} + +func (s *Service) resolveRestoreSelector(ctx context.Context, input RestorePlanInput) (int64, RestoreSourceRef, error) { + switch input.Selector { + case RestoreByFileID: + return input.FileID, RestoreSourceRef{}, nil + case RestoreByStoredPath: + return s.resolveStoredPath(ctx, input.StoredPath) + case RestoreBySnapshotPath: + return s.resolveSnapshotPath(ctx, input.SnapshotID, input.SnapshotPath) + default: + return 0, RestoreSourceRef{}, NewError(ErrorInvalidArgument, "load restore plan", "exactly_one_restore_selector", "unsupported restore selector", nil) + } +} + +func (s *Service) resolveStoredPath(ctx context.Context, path string) (int64, RestoreSourceRef, error) { + var logicalID int64 + var mode, uid, gid sql.NullInt64 + var mtime sql.NullTime + var complete bool + err := s.db.QueryRowContext(ctx, ` +SELECT logical_file_id, mode, mtime, uid, gid, is_metadata_complete +FROM physical_file WHERE path = $1`, path).Scan(&logicalID, &mode, &mtime, &uid, &gid, &complete) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, RestoreSourceRef{}, NewError(ErrorNotFound, "load restore plan", "", fmt.Sprintf("stored path %q not found", path), err) + } + return 0, RestoreSourceRef{}, restoreCatalogError(fmt.Errorf("resolve stored path %q: %w", path, err)) + } + return logicalID, RestoreSourceRef{StoredPath: path, Mode: nullInt64Pointer(mode), MTime: nullTimePointer(mtime), UID: nullInt64Pointer(uid), GID: nullInt64Pointer(gid), IsMetadataComplete: complete}, nil +} + +func (s *Service) resolveSnapshotPath(ctx context.Context, snapshotID, path string) (int64, RestoreSourceRef, error) { + var logicalID int64 + var size, mode sql.NullInt64 + var mtime sql.NullTime + err := s.db.QueryRowContext(ctx, ` +SELECT sf.logical_file_id, sf.size, sf.mode, sf.mtime +FROM snapshot_file sf +JOIN snapshot_path sp ON sp.id = sf.path_id +WHERE sf.snapshot_id = $1 AND sp.path = $2`, snapshotID, path).Scan(&logicalID, &size, &mode, &mtime) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, RestoreSourceRef{}, NewError(ErrorNotFound, "load restore plan", "", fmt.Sprintf("snapshot path %q in snapshot %q not found", path, snapshotID), err) + } + return 0, RestoreSourceRef{}, restoreCatalogError(fmt.Errorf("resolve snapshot path %q in snapshot %q: %w", path, snapshotID, err)) + } + return logicalID, RestoreSourceRef{SnapshotID: snapshotID, SnapshotPath: path, Size: nullInt64Pointer(size), Mode: nullInt64Pointer(mode), MTime: nullTimePointer(mtime), IsMetadataComplete: size.Valid && mode.Valid && mtime.Valid}, nil +} + +func (s *Service) loadRestoreLogicalFile(ctx context.Context, id int64) (RestoreLogicalFileRef, error) { + var ref RestoreLogicalFileRef + err := s.db.QueryRowContext(ctx, ` +SELECT id, original_name, total_size, file_hash, chunker_version, status +FROM logical_file WHERE id = $1`, id).Scan(&ref.ID, &ref.OriginalName, &ref.TotalSize, &ref.FileHash, &ref.ChunkerVersion, &ref.Status) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return RestoreLogicalFileRef{}, NewError(ErrorNotFound, "load restore plan", "", fmt.Sprintf("logical file %d not found", id), err) + } + return RestoreLogicalFileRef{}, restoreCatalogError(fmt.Errorf("load logical file %d: %w", id, err)) + } + if ref.OriginalName == "" || ref.FileHash == "" || ref.ChunkerVersion == "" || ref.TotalSize < 0 { + return RestoreLogicalFileRef{}, NewError(ErrorInvariantViolation, "load restore plan", "complete_logical_file_metadata", fmt.Sprintf("logical file %d metadata is incomplete", id), nil) + } + return ref, nil +} + +func cloneRestoreSelector(input RestorePlanInput) RestorePlanInput { return input } + +func nullInt64Pointer(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + converted := value.Int64 + return &converted +} + +func nullTimePointer(value sql.NullTime) *time.Time { + if !value.Valid { + return nil + } + converted := value.Time.UTC() + return &converted +} + +func restoreCatalogError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return NewError(ErrorCancelled, "load restore plan", "", "restore plan load cancelled", err) + } + return NewError(ErrorOperationFailed, "load restore plan", "", "restore plan query failed", err) } diff --git a/internal/catalog/service_test.go b/internal/catalog/service_test.go index 80c4ce24..da601b58 100644 --- a/internal/catalog/service_test.go +++ b/internal/catalog/service_test.go @@ -389,25 +389,14 @@ func TestServiceLoadReachabilityRoots(t *testing.T) { assertReachabilityRoots(t, roots) } -// TestServiceDeferredMethodsReturnErrNotImplemented verifies the remaining -// restore/GC skeleton methods return ErrNotImplemented. +// TestServiceDeferredMethodsReturnErrNotImplemented verifies the remaining GC +// skeleton method returns ErrNotImplemented. func TestServiceDeferredMethodsReturnErrNotImplemented(t *testing.T) { dbconn := openTestDB(t) svc := catalog.NewServiceFromSQL(dbconn) ctx := context.Background() before := countCatalogLogicalFiles(t, dbconn) - restorePlan, err := svc.LoadRestorePlanMetadata(ctx, catalog.RestorePlanInput{FileID: 1}) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadRestorePlanMetadata: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadRestorePlanMetadata: want catalog.IsDeferred=true, got %v", err) - } - if restorePlan != nil { - t.Errorf("LoadRestorePlanMetadata: want nil metadata on deferred path, got %+v", restorePlan) - } - gcPlan, err := svc.LoadGCPlanMetadata(ctx, catalog.GCPlanInput{}) if !errors.Is(err, catalog.ErrNotImplemented) { t.Errorf("LoadGCPlanMetadata: want ErrNotImplemented via errors.Is, got %v", err) diff --git a/internal/catalog/types.go b/internal/catalog/types.go index 4d23f3e0..c6b3420a 100644 --- a/internal/catalog/types.go +++ b/internal/catalog/types.go @@ -160,6 +160,7 @@ type RestoreSourceRef struct { StoredPath string SnapshotID string SnapshotPath string + Size *int64 Mode *int64 MTime *time.Time UID *int64 From 747cba2fbd6e2dbea322d76f0794f7e1a7396d33 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 13:29:48 +0200 Subject: [PATCH 09/27] Complete v1.13.12 restore plan adoption --- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...1.13.12-phase6-placement-implementation.md | 20 +- ...3.12-phase8-restore-production-adoption.md | 69 ++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 23 +- internal/catalog/contracts.go | 20 +- internal/catalog/placement.go | 8 - internal/catalog/restore_plan.go | 19 +- internal/storage/restore.go | 218 +++++------------- .../storage/restore_catalog_adoption_test.go | 138 +++++++++++ 10 files changed, 334 insertions(+), 193 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase8-restore-production-adoption.md create mode 100644 internal/storage/restore_catalog_adoption_test.go diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 1133e30d..e2c0038d 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -57,11 +57,13 @@ ## Phase 8 — Restore Production Adoption -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 8 restore adoption](v1.13.12-phase8-restore-production-adoption.md). ## Phase 9 — GC-Plan Implementation and Adoption -**Status:** Not started +**Status:** Next ## Phase 10 — Folder and Recursive Store Ownership diff --git a/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md b/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md index 80d150ef..13a4e018 100644 --- a/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md +++ b/docs/release/v1.13/v1.13.12-phase6-placement-implementation.md @@ -27,11 +27,17 @@ packed variant only. It falls back to the legacy variant only when no packed reference exists. The implementation fails closed, with no partial recipe, on missing placement -metadata, non-contiguous ordering, invalid codec/nonce/compression contracts, -out-of-bounds container or segment ranges, missing mandatory block hashes, -integer overflow, or a recipe-size/logical-file-size mismatch. Quarantine is -preserved as catalog truth rather than erased; restore execution remains -responsible for its existing refusal behavior. +metadata, non-contiguous ordering, empty provenance, invalid tagged-union +shape, invalid codec/nonce/compression contracts, and structurally invalid +container or segment metadata. Quarantine is preserved as catalog truth rather +than erased. + +Phase 8 compatibility testing clarified the validation boundary: physical +container bounds, block-hash presence and comparison, decoded-size checks, and +whole-recipe byte accounting remain owned by the established restore reader. +The catalog carries those values without pre-empting the reader's frozen error +classification or failure order. This correction does not weaken restore +validation; it prevents the metadata planner from replacing it. ## Compatibility and parity @@ -52,8 +58,8 @@ output, exit code, or backend default changed. - Shared mixed legacy/packed placement and restore-query parity contract on SQLite and PostgreSQL: pass. - Missing file, invalid ID, zero-length file, missing placement, cancellation, - deterministic repeat, no-mutation, tagged-union, bounds, and transform tests: - pass. + deterministic repeat, no-mutation, tagged-union, structural-metadata, and + transform tests: pass. - Validation root: `/tmp/coldkeep-v11312-phase6.3BxWXO/`, with all Go caches and unique suite storage roots below it. diff --git a/docs/release/v1.13/v1.13.12-phase8-restore-production-adoption.md b/docs/release/v1.13/v1.13.12-phase8-restore-production-adoption.md new file mode 100644 index 00000000..d908f8d0 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase8-restore-production-adoption.md @@ -0,0 +1,69 @@ +# Coldkeep v1.13.12 Phase 8 — Restore Production Adoption + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `8 — Restore Production Adoption` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Production adoption + +The production restore pin/load transaction now resolves one immutable +`catalog.RestorePlanMetadata` by logical-file ID through the transaction-bound +catalog service. The same transaction then increments the plan's ordered chunk +IDs and commits before byte reads, preserving the existing GC-protection +lifetime and rollback behavior. + +The storage adapter projects neutral legacy or packed placement metadata into +the established internal restore recipe. Existing byte I/O, dual-layout chunk +resolution, container reads, integrity verification, pin cleanup, temporary +output, destination validation, and atomic rename behavior are unchanged. + +The former production logical-file query and joined recipe query were removed. +A test-only copy of the old query is retained solely as a parity oracle. It +proves common execution metadata and hashes for legacy and packed layouts, +exact legacy fields, deterministic chunk IDs, and pin/unpin restoration. + +## Frozen compatibility boundary + +Initial full-storage validation showed that catalog-side physical-bounds and +whole-recipe-size checks could reject historical/adversarial fixtures before +the established restore reader. The boundary was corrected without weakening +validation: + +- catalog owns selector truth, completed-file state, ordered recipe structure, + placement shape, provenance presence, and neutral metadata; +- the restore reader owns physical container/block bounds, required hash and + hash comparison checks, decompression/decoding sizes, byte accounting, and + their frozen failure order. + +Compatibility translation at the storage adapter preserves established +not-found, no-restorable-chunks, ordering, and chunker-version messages. No +schema, migration, persisted metadata, storage byte, lock, backend default, +CLI/JSON shape, or exit code changed. + +## Validation + +- Focused compression, hash, malformed metadata, no-recipe, incomplete-chunk, + ordering, legacy, packed, and parity fixtures: pass. +- `go test -count=1 ./internal/storage`: pass. +- `go test -count=1 ./internal/catalog`: pass. +- `go test -count=1 ./internal/engine ./cmd/coldkeep`: pass. +- `go test -race -count=1 ./internal/catalog ./internal/storage`: pass. +- Shared restore-plan contract on SQLite and PostgreSQL: pass. +- Validation root: `/tmp/coldkeep-v11312-phase8.mch5th/`, with caches, temp + files, storage roots, and test artifacts isolated below it. + +The first PostgreSQL attempt could not open a sandboxed local socket and +created no database. The next attempt created +`coldkeep_bt_testcatalogcontractrestoreplansacrossbacke_2798d6_1`, correctly +refused an uninitialized schema, and dropped it. The final auto-bootstrapped +contract pass created and dropped +`coldkeep_bt_testcatalogcontractrestoreplansacrossbacke_279a21_1`. A read-only +maintenance query confirmed both exact names are absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 9 is authorized to implement and adopt catalog GC reachability planning. +It may not change deletion, lock, transaction, dry-run, fail-fast, rollback, +reporting, or live-GC exclusion semantics. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 1f5dee6d..3c9d2b9e 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–7 are Complete. Phase 8 is Next: adopt the catalog restore plan in the -production storage path while preserving pinning, transaction lifetime, -integrity checks, byte I/O, and atomic destination behavior. +Phases 0–8 are Complete. Phase 9 is Next: implement and adopt catalog-owned GC +reachability planning while preserving deletion, locking, transaction, +dry-run, fail-fast, rollback, reporting, and live-GC exclusion semantics. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index c8f6e2b5..8663cd0b 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -93,8 +93,10 @@ SQLite and PostgreSQL. - [x] Packed references take authority over compatibility companion rows and exactly one tagged placement is returned per chunk. -- [x] Codec, nonce, compression, hash, segment, container, order, overflow, and - total recipe-size invariants fail closed with no partial result. +- [x] Codec, nonce, compression, tagged-union, structural segment/container, + and order invariants fail closed with no partial result. +- [x] Physical bounds, hashes, decoded sizes, and whole-recipe byte accounting + remain in the restore reader so frozen error classification/order is intact. - [x] Zero-length, missing, malformed, cancelled, deterministic, and read-only behaviors are directly tested. - [x] New placement fields have parity with the existing restore metadata and @@ -115,11 +117,24 @@ ## Phase 8 — Restore Production Adoption -**Phase status:** Next +**Phase status:** Complete + +- [x] Production restore metadata and pinning use the catalog restore plan in + the existing caller-owned transaction. +- [x] Legacy and packed catalog projections match the removed query's + execution metadata and deterministic pinned-ID ordering. +- [x] The duplicated production restore query is removed; its shape remains a + test-only parity oracle. +- [x] Byte I/O, resolver selection, integrity checks, pin lifetime, atomic + destination behavior, frozen errors, and public output remain unchanged. +- [x] SQLite/PostgreSQL plan parity, full storage tests, focused engine/CLI + suites, and catalog/storage race tests pass under isolated roots. +- [x] Exact Phase 8 PostgreSQL scratch databases were dropped and confirmed + absent. ## Phase 9 — GC-Plan Implementation and Adoption -**Phase status:** Not started +**Phase status:** Next ## Phase 10 — Folder and Recursive Store Ownership diff --git a/internal/catalog/contracts.go b/internal/catalog/contracts.go index 2ee6dc04..f3a129a4 100644 --- a/internal/catalog/contracts.go +++ b/internal/catalog/contracts.go @@ -41,9 +41,12 @@ func NormalizeGCPlanInput(input GCPlanInput) (GCPlanInput, error) { // ValidateChunkPlacement enforces the tagged union and shared range invariants. func ValidateChunkPlacement(placement ChunkPlacementRef) error { - if placement.ChunkOrder < 0 || placement.ChunkID <= 0 || placement.ChunkHash == "" || placement.ChunkSize <= 0 || placement.ChunkerVersion == "" || placement.ChunkStatus == "" { + if placement.ChunkOrder < 0 || placement.ChunkID <= 0 || placement.ChunkHash == "" || placement.ChunkSize <= 0 || placement.ChunkStatus == "" { return invalidPlacement(placement, "chunk identity and order must be complete") } + if strings.TrimSpace(placement.ChunkerVersion) == "" { + return invalidPlacement(placement, "chunk has empty chunker_version (repository corruption or incomplete migration)") + } switch placement.Kind { case PlacementLegacy: if placement.Legacy == nil || placement.Packed != nil { @@ -52,18 +55,18 @@ func ValidateChunkPlacement(placement ChunkPlacementRef) error { if placement.Legacy.BlockID <= 0 || placement.Legacy.Codec == "" || placement.Legacy.FormatVersion <= 0 || placement.Legacy.PlaintextSize <= 0 || placement.Legacy.StoredSize <= 0 || !validContainerRead(placement.Legacy.Container, placement.Legacy.ContainerOffset, placement.Legacy.StoredSize) { return invalidPlacement(placement, "legacy block and container bounds must be valid") } - if placement.Legacy.PlaintextSize != placement.ChunkSize || !validLegacyCodec(placement.Legacy.Codec, placement.Legacy.Nonce) { - return invalidPlacement(placement, "legacy codec, nonce, and plaintext size must match the chunk") + if !validLegacyCodec(placement.Legacy.Codec, placement.Legacy.Nonce) { + return invalidPlacement(placement, "legacy codec and nonce must be valid") } case PlacementPacked: if placement.Packed == nil || placement.Legacy != nil { return invalidPlacement(placement, "packed placement must contain only packed metadata") } - if placement.Packed.BlockID <= 0 || placement.Packed.Codec == "" || placement.Packed.FormatVersion <= 0 || placement.Packed.PlaintextSize <= 0 || placement.Packed.CompressionCodec == "" || placement.Packed.StoredSize <= 0 || len(placement.Packed.BlockHash) == 0 || placement.Packed.OffsetInBlock < 0 || placement.Packed.SizeInBlock <= 0 || placement.Packed.OffsetInBlock+placement.Packed.SizeInBlock > placement.Packed.PlaintextSize || !validContainerRead(placement.Packed.Container, placement.Packed.ContainerOffset, placement.Packed.StoredSize) { - return invalidPlacement(placement, "packed block, segment, and container bounds must be valid") + if placement.Packed.BlockID <= 0 || placement.Packed.Codec == "" || placement.Packed.FormatVersion <= 0 || placement.Packed.PlaintextSize <= 0 || placement.Packed.CompressionCodec == "" || placement.Packed.StoredSize <= 0 || placement.Packed.OffsetInBlock < 0 || placement.Packed.SizeInBlock <= 0 || !validContainerRead(placement.Packed.Container, placement.Packed.ContainerOffset, placement.Packed.StoredSize) { + return invalidPlacement(placement, "packed block, segment, and container metadata must be valid") } - if placement.Packed.SizeInBlock != placement.ChunkSize || !validPackedTransforms(placement.Packed) { - return invalidPlacement(placement, "packed codec, compression, and segment size must be valid") + if !validPackedTransforms(placement.Packed) { + return invalidPlacement(placement, "packed codec and compression metadata must be valid") } default: return invalidPlacement(placement, "placement kind must be legacy or packed") @@ -100,8 +103,7 @@ func validContainerRead(container ContainerPlacementRef, offset, size int64) boo if container.ID <= 0 || container.Filename == "" || offset < 0 || size <= 0 || container.CurrentSize < 0 || container.MaxSize <= 0 { return false } - end := offset + size - return end >= offset && end <= container.CurrentSize && end <= container.MaxSize + return true } func invalidPlacement(placement ChunkPlacementRef, message string) error { diff --git a/internal/catalog/placement.go b/internal/catalog/placement.go index e58cb742..ffb7d854 100644 --- a/internal/catalog/placement.go +++ b/internal/catalog/placement.go @@ -38,7 +38,6 @@ func (s *Service) LoadChunkPlacements(ctx context.Context, logicalFileID int64) placements := make([]ChunkPlacementRef, 0) var expectedOrder int64 - var recipeSize int64 for rows.Next() { scanned, err := scanChunkPlacement(rows) if err != nil { @@ -55,10 +54,6 @@ func (s *Service) LoadChunkPlacements(ctx context.Context, logicalFileID int64) return nil, err } placements = append(placements, placement) - if placement.ChunkSize > 0 && recipeSize > int64(^uint64(0)>>1)-placement.ChunkSize { - return nil, NewError(ErrorInvariantViolation, "load chunk placements", "recipe_size_no_overflow", fmt.Sprintf("logical file %d chunk sizes overflow int64", logicalFileID), nil) - } - recipeSize += placement.ChunkSize expectedOrder++ } if err := rows.Err(); err != nil { @@ -67,9 +62,6 @@ func (s *Service) LoadChunkPlacements(ctx context.Context, logicalFileID int64) if len(placements) == 0 && totalSize != 0 { return nil, NewError(ErrorInvariantViolation, "load chunk placements", "nonempty_file_has_recipe", fmt.Sprintf("logical file %d has size %d but no chunk recipe", logicalFileID, totalSize), nil) } - if recipeSize != totalSize { - return nil, NewError(ErrorInvariantViolation, "load chunk placements", "recipe_size_matches_logical_file", fmt.Sprintf("logical file %d recipe size %d does not match total size %d", logicalFileID, recipeSize, totalSize), nil) - } return placements, nil } diff --git a/internal/catalog/restore_plan.go b/internal/catalog/restore_plan.go index 257ee42e..86a9b6a5 100644 --- a/internal/catalog/restore_plan.go +++ b/internal/catalog/restore_plan.go @@ -5,9 +5,13 @@ import ( "database/sql" "errors" "fmt" + "regexp" + "strings" "time" ) +var restoreChunkerVersionPattern = regexp.MustCompile(`^v[0-9]+(?:-[a-z0-9]+)+$`) + // LoadRestorePlanMetadata resolves one selector and constructs a complete // immutable recipe using the Service's injected DB or caller-owned transaction. func (s *Service) LoadRestorePlanMetadata(ctx context.Context, input RestorePlanInput) (*RestorePlanMetadata, error) { @@ -29,13 +33,13 @@ func (s *Service) LoadRestorePlanMetadata(ctx context.Context, input RestorePlan if err != nil { return nil, err } + if logical.Status != "COMPLETED" { + return nil, NewError(ErrorConflict, "load restore plan", "completed_logical_file", fmt.Sprintf("logical file %d is not completed", logicalID), nil) + } placements, err := s.LoadChunkPlacements(ctx, logicalID) if err != nil { return nil, err } - if logical.Status != "COMPLETED" { - return nil, NewError(ErrorConflict, "load restore plan", "completed_logical_file", fmt.Sprintf("logical file %d is not completed", logicalID), nil) - } if (logical.TotalSize == 0 && len(placements) != 0) || (logical.TotalSize > 0 && len(placements) == 0) { return nil, NewError(ErrorInvariantViolation, "load restore plan", "recipe_matches_logical_file_size", fmt.Sprintf("logical file %d has inconsistent zero-length recipe metadata", logicalID), nil) } @@ -101,9 +105,16 @@ FROM logical_file WHERE id = $1`, id).Scan(&ref.ID, &ref.OriginalName, &ref.Tota } return RestoreLogicalFileRef{}, restoreCatalogError(fmt.Errorf("load logical file %d: %w", id, err)) } - if ref.OriginalName == "" || ref.FileHash == "" || ref.ChunkerVersion == "" || ref.TotalSize < 0 { + if ref.OriginalName == "" || ref.FileHash == "" || ref.TotalSize < 0 { return RestoreLogicalFileRef{}, NewError(ErrorInvariantViolation, "load restore plan", "complete_logical_file_metadata", fmt.Sprintf("logical file %d metadata is incomplete", id), nil) } + trimmedChunkerVersion := strings.TrimSpace(ref.ChunkerVersion) + if trimmedChunkerVersion == "" { + return RestoreLogicalFileRef{}, NewError(ErrorInvariantViolation, "load restore plan", "complete_logical_file_metadata", fmt.Sprintf("logical file %d has empty chunker_version (repository corruption or incomplete migration)", id), nil) + } + if !restoreChunkerVersionPattern.MatchString(trimmedChunkerVersion) { + return RestoreLogicalFileRef{}, NewError(ErrorInvariantViolation, "load restore plan", "complete_logical_file_metadata", fmt.Sprintf("logical file %d has malformed chunker_version %q (expected format like v1-simple-rolling)", id, ref.ChunkerVersion), nil) + } return ref, nil } diff --git a/internal/storage/restore.go b/internal/storage/restore.go index 1815179a..118059c7 100644 --- a/internal/storage/restore.go +++ b/internal/storage/restore.go @@ -16,6 +16,7 @@ import ( "strings" "github.com/franchoy/coldkeep/internal/blocks" + "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/chunk" "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/db" @@ -162,17 +163,6 @@ type restoreChunkRow struct { chunkID int64 } -type restoreLogicalFileRow struct { - id int64 - originalName string - totalSize int64 - fileHash string - status string - // chunkerVersion identifies the provenance of the persisted file recipe. - // It is metadata about how the logical recipe was originally produced. - chunkerVersion string -} - // ================================================================ // Internal semantic restore recipe types (Phase 6) // ================================================================ @@ -465,36 +455,6 @@ func buildBlockReadPlan(chunkSegments []*blocks.ChunkSegment) *blockReadPlan { return plan } -func loadCompletedLogicalFileRowForRestore(ctx context.Context, tx *sql.Tx, fileID int64) (restoreLogicalFileRow, error) { - var row restoreLogicalFileRow - err := tx.QueryRowContext( - ctx, - `SELECT id, original_name, total_size, file_hash, status, chunker_version - FROM logical_file - WHERE status = $1 AND id = $2`, - filestate.LogicalFileCompleted, - fileID, - ).Scan( - &row.id, - &row.originalName, - &row.totalSize, - &row.fileHash, - &row.status, - &row.chunkerVersion, - ) - if err == sql.ErrNoRows { - return restoreLogicalFileRow{}, fmt.Errorf("logical file id %d not found", fileID) - } - if err != nil { - return restoreLogicalFileRow{}, fmt.Errorf("query logical_file: %w", err) - } - if err := validateRestoreLogicalFileChunkerVersion(fileID, row.chunkerVersion); err != nil { - return restoreLogicalFileRow{}, err - } - - return row, nil -} - func pinLogicalFileRestoreChunks(dbconn *sql.DB, fileID int64) (string, string, []restoreChunkRow, []int64, error) { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() @@ -512,125 +472,37 @@ func pinLogicalFileRestoreChunksWithContext(ctx context.Context, dbconn *sql.DB, } }() - logicalFileRow, err := loadCompletedLogicalFileRowForRestore(ctx, tx, fileID) + plan, err := catalog.NewService(tx).LoadRestorePlanMetadata(ctx, catalog.RestorePlanInput{ + Selector: catalog.RestoreByFileID, + FileID: fileID, + }) if err != nil { - return "", "", nil, nil, err - } - - // ================================================================ - // DESIGN PRINCIPLE: One ordered chunk recipe query per file - // ================================================================ - // This is a performance-critical optimization: - // - Single query loads ALL chunk metadata for the file (ordered by chunk_order) - // - Result includes: offsets, sizes, hashes, codecs, container locations - // - Loop below iterates pre-loaded rows with NO additional DB queries - // - This ensures O(1) DB operations per file, not O(n) per chunk - // - // DO NOT refactor this into: - // - per-chunk lookup loops (n queries per file x files) - // - lazy-load patterns (defeats tuple prefetching, adds latency) - // - separate queries for offset/hash/codec (cache-unfriendly) - // - // STAGE 2-4: Pin chunks + load metadata/recipe (atomic transaction) - // ================================================================ - // Query: ordered chunks for this logical file + blocks metadata - // Action: INCREMENT pin_count for each chunk (GC protection) - // Result: snapshot of deterministic chunk recipe + pinned IDs - // Guarantee: ✓ If commit succeeds, chunks are pinned and cannot be GC'd - // Guarantee: ✓ Query is ordered by chunk_order for deterministic restore - // - rows, err := tx.QueryContext(ctx, ` - SELECT - fc.chunk_order, - COALESCE(b.block_offset, 0), - COALESCE(b.plaintext_size, c.size), - COALESCE(b.stored_size, c.size), - c.chunk_hash, - sb.block_hash, - sb.compressed_hash, - sb.physical_hash, - c.chunker_version, - c.size, - COALESCE(b.codec, 'plain'), - COALESCE(b.format_version, 1), - b.nonce, - COALESCE(b.container_id, 0), - ctr.filename, - c.status, - ctr.max_size, - c.id - FROM file_chunk fc - JOIN chunk c ON c.id = fc.chunk_id - LEFT JOIN blocks b ON b.chunk_id = c.id - LEFT JOIN chunk_block_refs r ON r.chunk_id = c.id - LEFT JOIN storage_blocks sb ON sb.id = r.block_id - LEFT JOIN container ctr ON ctr.id = COALESCE(b.container_id, sb.container_id) - WHERE fc.logical_file_id = $1 AND c.status = $2 - ORDER BY fc.chunk_order ASC - `, fileID, filestate.ChunkCompleted) - if err != nil { - return "", "", nil, nil, fmt.Errorf("query file chunks: %w", err) - } - defer func() { _ = rows.Close() }() - - chunkRows := make([]restoreChunkRow, 0) - pinnedChunkIDs := make([]int64, 0) - for rows.Next() { - var row restoreChunkRow - if err := rows.Scan( - &row.chunkOrder, - &row.blockOffset, - &row.plaintextSize, - &row.storedSize, - &row.expectedChunkHash, - &row.blockHash, - &row.compressedHash, - &row.physicalHash, - &row.chunkerVersion, - &row.chunkSize, - &row.blocksCodec, - &row.blocksFormatVersion, - &row.blocksNonce, - &row.blocksContainerID, - &row.filename, - &row.chunkStatus, - &row.maxSize, - &row.chunkID, - ); err != nil { - return "", "", nil, nil, fmt.Errorf("scan chunk row: %w", err) + if catalog.IsCode(err, catalog.ErrorNotFound) || catalog.IsCode(err, catalog.ErrorConflict) { + return "", "", nil, nil, fmt.Errorf("logical file id %d not found", fileID) } - trimmedChunkVersion := strings.TrimSpace(row.chunkerVersion) - if trimmedChunkVersion == "" { - return "", "", nil, nil, fmt.Errorf("chunk %d has empty chunker_version (repository corruption or incomplete migration)", row.chunkID) - } - if !chunk.IsWellFormedVersion(chunk.Version(trimmedChunkVersion)) { - return "", "", nil, nil, fmt.Errorf("chunk %d has malformed chunker_version %q (expected format like v1-simple-rolling)", row.chunkID, trimmedChunkVersion) - } - // Phase 4 compatibility rule: restore only requires chunk-level version - // metadata sanity/presence. It must not enforce per-file equality between - // logical_file.chunker_version and chunk.chunker_version because chunk rows - // are content-addressed and can be legitimately reused across version eras. - // - // chunk.chunker_version is origin metadata for the chunk row, not a restore - // compatibility constraint for every logical file that references it. - // If the container is missing (quarantined), filename will be NULL - // Allow the chunk row, but mark filename as empty string - if row.filename == "" { - row.filename = "" + var catalogErr *catalog.Error + if errors.As(err, &catalogErr) { + if strings.Contains(catalogErr.Message, "empty chunker_version") { + return "", "", nil, nil, errors.New(catalogErr.Message) + } + switch catalogErr.Invariant { + case "nonempty_file_has_recipe", "exactly_one_valid_placement_per_chunk": + return "", "", nil, nil, fmt.Errorf("no restorable chunks found for file %d (all referenced containers missing or quarantined)", fileID) + case "contiguous_chunk_order": + return "", "", nil, nil, fmt.Errorf("invalid restore recipe ordering: non-contiguous restore chunk order: %s", catalogErr.Message) + } } - chunkRows = append(chunkRows, row) - pinnedChunkIDs = append(pinnedChunkIDs, row.chunkID) + return "", "", nil, nil, fmt.Errorf("load catalog restore plan for logical file %d: %w", fileID, err) } - if err := rows.Err(); err != nil { - return "", "", nil, nil, fmt.Errorf("iterate chunk rows: %w", err) + if err := validateRestoreLogicalFileChunkerVersion(fileID, plan.LogicalFile.ChunkerVersion); err != nil { + return "", "", nil, nil, err + } + chunkRows, pinnedChunkIDs, err := restoreRowsFromCatalogPlan(plan) + if err != nil { + return "", "", nil, nil, err } - for _, chunkID := range pinnedChunkIDs { - result, execErr := tx.ExecContext( - ctx, - `UPDATE chunk SET pin_count = pin_count + 1 WHERE id = $1`, - chunkID, - ) + result, execErr := tx.ExecContext(ctx, `UPDATE chunk SET pin_count = pin_count + 1 WHERE id = $1`, chunkID) if execErr != nil { return "", "", nil, nil, fmt.Errorf("pin chunk %d for restore: %w", chunkID, execErr) } @@ -642,13 +514,47 @@ func pinLogicalFileRestoreChunksWithContext(ctx context.Context, dbconn *sql.DB, return "", "", nil, nil, fmt.Errorf("chunk %d disappeared while pinning restore", chunkID) } } - if err := tx.Commit(); err != nil { return "", "", nil, nil, err } tx = nil + return plan.LogicalFile.OriginalName, plan.LogicalFile.FileHash, chunkRows, pinnedChunkIDs, nil +} - return logicalFileRow.originalName, logicalFileRow.fileHash, chunkRows, pinnedChunkIDs, nil +func restoreRowsFromCatalogPlan(plan *catalog.RestorePlanMetadata) ([]restoreChunkRow, []int64, error) { + if plan == nil { + return nil, nil, errors.New("catalog restore plan is nil") + } + rows := make([]restoreChunkRow, 0, len(plan.Placements)) + pinned := make([]int64, 0, len(plan.Placements)) + for _, placement := range plan.Placements { + trimmedVersion := strings.TrimSpace(placement.ChunkerVersion) + if trimmedVersion == "" { + return nil, nil, fmt.Errorf("chunk %d has empty chunker_version (repository corruption or incomplete migration)", placement.ChunkID) + } + if !chunk.IsWellFormedVersion(chunk.Version(trimmedVersion)) { + return nil, nil, fmt.Errorf("chunk %d has malformed chunker_version %q (expected format like v1-simple-rolling)", placement.ChunkID, trimmedVersion) + } + row := restoreChunkRow{chunkOrder: placement.ChunkOrder, expectedChunkHash: placement.ChunkHash, chunkerVersion: placement.ChunkerVersion, chunkSize: placement.ChunkSize, chunkStatus: placement.ChunkStatus, chunkID: placement.ChunkID} + switch placement.Kind { + case catalog.PlacementLegacy: + legacy := placement.Legacy + row.blockOffset, row.plaintextSize, row.storedSize = legacy.ContainerOffset, legacy.PlaintextSize, legacy.StoredSize + row.blocksCodec, row.blocksFormatVersion, row.blocksNonce = legacy.Codec, legacy.FormatVersion, append([]byte(nil), legacy.Nonce...) + row.blocksContainerID, row.filename, row.maxSize = legacy.Container.ID, legacy.Container.Filename, legacy.Container.MaxSize + case catalog.PlacementPacked: + packed := placement.Packed + row.blockOffset, row.plaintextSize, row.storedSize = packed.ContainerOffset, placement.ChunkSize, packed.StoredSize + row.blockHash, row.compressedHash, row.physicalHash = append([]byte(nil), packed.BlockHash...), append([]byte(nil), packed.CompressedHash...), append([]byte(nil), packed.PhysicalHash...) + row.blocksCodec, row.blocksFormatVersion = packed.Codec, packed.FormatVersion + row.blocksContainerID, row.filename, row.maxSize = packed.Container.ID, packed.Container.Filename, packed.Container.MaxSize + default: + return nil, nil, fmt.Errorf("chunk %d has unsupported catalog placement %q", placement.ChunkID, placement.Kind) + } + rows = append(rows, row) + pinned = append(pinned, placement.ChunkID) + } + return rows, pinned, nil } func unpinRestoreChunks(dbconn *sql.DB, chunkIDs []int64) error { diff --git a/internal/storage/restore_catalog_adoption_test.go b/internal/storage/restore_catalog_adoption_test.go new file mode 100644 index 00000000..be6fbcef --- /dev/null +++ b/internal/storage/restore_catalog_adoption_test.go @@ -0,0 +1,138 @@ +package storage + +import ( + "bytes" + "context" + "database/sql" + "reflect" + "testing" + + "github.com/franchoy/coldkeep/internal/catalog" + filestate "github.com/franchoy/coldkeep/internal/status" +) + +func TestCatalogRestorePlanPreservesLegacyQueryExecutionMetadata(t *testing.T) { + for _, tc := range []struct { + name string + packed bool + }{ + {name: "legacy"}, + {name: "packed", packed: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dbconn := setupStep8DB(t) + defer func() { _ = dbconn.Close() }() + containersDir := t.TempDir() + chunks := []restoreChunkSeed{ + insertChunkForRestore(t, dbconn, []byte("catalog-adoption-a"), "v1-simple-rolling"), + insertChunkForRestore(t, dbconn, []byte("catalog-adoption-b"), "v1-simple-rolling"), + } + fileID := insertLogicalFileForRestore(t, dbconn, tc.name+".bin", chunks, "v1-simple-rolling") + if tc.packed { + insertPackedStorageBlock(t, dbconn, containersDir, "packed.ckc", chunks) + } else { + containerID := insertContainerWithPayload(t, dbconn, containersDir, "legacy.ckc", chunks[0].payload) + insertLegacyBlocksRows(t, dbconn, containerID, chunks) + } + + legacyRows := loadLegacyRestoreRowsForParity(t, dbconn, fileID) + plan, err := catalog.NewServiceFromSQL(dbconn).LoadRestorePlanMetadata(context.Background(), catalog.RestorePlanInput{Selector: catalog.RestoreByFileID, FileID: fileID}) + if err != nil { + t.Fatalf("load catalog restore plan: %v", err) + } + catalogRows, pinnedIDs, err := restoreRowsFromCatalogPlan(plan) + if err != nil { + t.Fatalf("map catalog restore plan: %v", err) + } + assertRestoreExecutionMetadataParity(t, legacyRows, catalogRows) + + wantIDs := []int64{chunks[0].id, chunks[1].id} + if !reflect.DeepEqual(pinnedIDs, wantIDs) { + t.Fatalf("pinned IDs=%v want=%v", pinnedIDs, wantIDs) + } + _, _, _, actualPinned, err := pinLogicalFileRestoreChunks(dbconn, fileID) + if err != nil { + t.Fatalf("pin catalog restore plan: %v", err) + } + if !reflect.DeepEqual(actualPinned, wantIDs) { + t.Fatalf("actual pinned IDs=%v want=%v", actualPinned, wantIDs) + } + if err := unpinRestoreChunks(dbconn, actualPinned); err != nil { + t.Fatalf("unpin catalog restore plan: %v", err) + } + for _, id := range wantIDs { + var count int64 + if err := dbconn.QueryRow(`SELECT pin_count FROM chunk WHERE id=$1`, id).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("chunk %d pin_count=%d after unpin", id, count) + } + } + }) + } +} + +func loadLegacyRestoreRowsForParity(t *testing.T, dbconn *sql.DB, fileID int64) []restoreChunkRow { + t.Helper() + rows, err := dbconn.Query(` +SELECT fc.chunk_order, COALESCE(b.block_offset,0), COALESCE(b.plaintext_size,c.size), + COALESCE(b.stored_size,c.size), c.chunk_hash, sb.block_hash, + sb.compressed_hash, sb.physical_hash, c.chunker_version, c.size, + COALESCE(b.codec,'plain'), COALESCE(b.format_version,1), b.nonce, + COALESCE(b.container_id,0), ctr.filename, c.status, ctr.max_size, c.id +FROM file_chunk fc +JOIN chunk c ON c.id=fc.chunk_id +LEFT JOIN blocks b ON b.chunk_id=c.id +LEFT JOIN chunk_block_refs r ON r.chunk_id=c.id +LEFT JOIN storage_blocks sb ON sb.id=r.block_id +LEFT JOIN container ctr ON ctr.id=COALESCE(b.container_id,sb.container_id) +WHERE fc.logical_file_id=$1 AND c.status=$2 +ORDER BY fc.chunk_order`, fileID, filestate.ChunkCompleted) + if err != nil { + t.Fatal(err) + } + defer func() { _ = rows.Close() }() + got := make([]restoreChunkRow, 0) + for rows.Next() { + var row restoreChunkRow + if err := rows.Scan(&row.chunkOrder, &row.blockOffset, &row.plaintextSize, + &row.storedSize, &row.expectedChunkHash, &row.blockHash, + &row.compressedHash, &row.physicalHash, &row.chunkerVersion, + &row.chunkSize, &row.blocksCodec, &row.blocksFormatVersion, + &row.blocksNonce, &row.blocksContainerID, &row.filename, + &row.chunkStatus, &row.maxSize, &row.chunkID); err != nil { + t.Fatal(err) + } + got = append(got, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return got +} + +func assertRestoreExecutionMetadataParity(t *testing.T, oldRows, newRows []restoreChunkRow) { + t.Helper() + if len(oldRows) != len(newRows) { + t.Fatalf("row count old=%d new=%d", len(oldRows), len(newRows)) + } + for i := range oldRows { + old, current := oldRows[i], newRows[i] + if old.chunkOrder != current.chunkOrder || old.chunkID != current.chunkID || + old.expectedChunkHash != current.expectedChunkHash || old.chunkerVersion != current.chunkerVersion || + old.chunkSize != current.chunkSize || old.chunkStatus != current.chunkStatus || + old.filename != current.filename || old.maxSize != current.maxSize || + !bytes.Equal(old.blockHash, current.blockHash) || + !bytes.Equal(old.compressedHash, current.compressedHash) || + !bytes.Equal(old.physicalHash, current.physicalHash) { + t.Fatalf("execution metadata mismatch row=%d old=%+v new=%+v", i, old, current) + } + if len(old.blockHash) == 0 && (old.blockOffset != current.blockOffset || + old.plaintextSize != current.plaintextSize || old.storedSize != current.storedSize || + old.blocksCodec != current.blocksCodec || old.blocksFormatVersion != current.blocksFormatVersion || + old.blocksContainerID != current.blocksContainerID || !bytes.Equal(old.blocksNonce, current.blocksNonce)) { + t.Fatalf("legacy execution metadata mismatch row=%d old=%+v new=%+v", i, old, current) + } + } +} From 8f8c01bc967cf9df1124f41a12103090178e5d6a Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 13:38:48 +0200 Subject: [PATCH 10/27] Complete v1.13.12 GC catalog planning --- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...se9-gc-plan-implementation-and-adoption.md | 61 ++++++++ docs/release/v1.13/v1.13.12-scope.md | 7 +- .../v1.13/v1.13.12-validation-checklist.md | 17 ++- internal/catalog/backend_contract_test.go | 78 ++++++---- internal/catalog/catalog.go | 13 +- internal/catalog/errors.go | 4 +- internal/catalog/gc_plan.go | 133 +++++++++++++++++- internal/catalog/service_test.go | 18 +-- internal/gc/plan.go | 43 ++---- internal/maintenance/gc.go | 29 +++- internal/observability/simulate.go | 8 +- 12 files changed, 319 insertions(+), 98 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index e2c0038d..e57131b7 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -63,11 +63,13 @@ ## Phase 9 — GC-Plan Implementation and Adoption -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 9 GC planning](v1.13.12-phase9-gc-plan-implementation-and-adoption.md). ## Phase 10 — Folder and Recursive Store Ownership -**Status:** Not started +**Status:** Next ## Phase 11 — List and Search Ownership diff --git a/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md b/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md new file mode 100644 index 00000000..d6c2e040 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md @@ -0,0 +1,61 @@ +# Coldkeep v1.13.12 Phase 9 — GC-Plan Implementation and Adoption + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `9 — GC-Plan Implementation and Adoption` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Catalog plan + +`catalog.Service.LoadGCPlanMetadata` now validates and normalizes excluded +snapshot IDs, loads protected snapshots in `created_at, id` order, and returns +logical-file roots in ID order. Each root records current-file ownership and +all retaining snapshots in the protected-snapshot order. + +Missing excluded snapshots return typed `not_found`; blank IDs return +`invalid_argument`; cancellation and query failures retain typed catalog +classification. Results are complete or nil, repeated reads are deterministic, +and the operation never mutates catalog state. + +## Production adoption + +All production GC root discovery now consumes the catalog plan: + +- `gc.BuildPlan` uses catalog roots before its existing graph traversal; +- live and dry-run maintenance GC use catalog roots for retention counters and + reachable-chunk marking; +- GC simulation uses the catalog for trace root counts and its shared plan. + +The previous `graph.Service.GCRoots` API remains available for historical graph +tests, but no production Go path calls it. Graph traversal from logical roots, +integrity preflight, advisory/session locking, sealed/active container checks, +packed-block liveness, deletion transactions, rollback, dry-run behavior, +failure order, and reporting are unchanged. + +All four promised catalog planning APIs are now implemented and production +consumed. The catalog deferred sentinel remains only as historical error +taxonomy compatibility; no active catalog method returns it. + +## Validation + +- `go test -count=1` for catalog, GC, maintenance, observability, engine, and + CLI packages: pass. +- Race tests for catalog, GC, maintenance, observability, and engine: pass. +- Shared catalog GC-plan contract on SQLite/PostgreSQL: pass. +- Shared engine GC dry-run contract on SQLite/PostgreSQL: pass. +- Existing orphan, protected/excluded snapshot, packed/legacy, rollback, + integrity-refusal, dry-run, and live-GC exclusion fixtures: pass. +- Validation root: `/tmp/coldkeep-v11312-phase9.Zl3gzQ/`, with isolated caches, + temp files, and per-suite storage roots. + +The catalog contract created and dropped exactly +`coldkeep_bt_testcatalogcontractgcplansacrossbackends_p_28824f_1`. The engine +GC contract created and dropped exactly +`coldkeep_bt_testenginegcdryrunacrossbackends_postgres_2883fd_1`. A read-only +maintenance query confirmed both names are absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 10 is authorized to add `Engine.StoreFolder` and migrate recursive/folder +orchestration without changing single-file storage semantics. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 3c9d2b9e..e6966d5b 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,10 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–8 are Complete. Phase 9 is Next: implement and adopt catalog-owned GC -reachability planning while preserving deletion, locking, transaction, -dry-run, fail-fast, rollback, reporting, and live-GC exclusion semantics. +Phases 0–9 are Complete. Phase 10 is Next: add a distinct engine-owned folder +store operation while preserving traversal, deterministic scheduling, worker, +warning, aggregate-result, stored-path, partial-failure, and finalization +semantics. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 8663cd0b..5da9f287 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -134,11 +134,24 @@ ## Phase 9 — GC-Plan Implementation and Adoption -**Phase status:** Next +**Phase status:** Complete + +- [x] Catalog returns deterministic current/snapshot logical roots and protected + snapshot metadata on SQLite and PostgreSQL. +- [x] Excluded snapshot IDs are normalized, validated, and removed without + mutating catalog state. +- [x] Real GC, dry-run planning, retention reporting, and GC simulation tracing + consume catalog root truth; no production path calls legacy graph discovery. +- [x] Existing graph traversal still computes reachable chunks from the neutral + catalog roots without changing sweep behavior. +- [x] Orphan, snapshot protection/exclusion, packed/legacy, rollback, + fail-closed integrity, and live-GC exclusion tests remain green. +- [x] Shared catalog and engine GC dry-run contracts pass on SQLite/PostgreSQL; + exact scratch databases were dropped and confirmed absent. ## Phase 10 — Folder and Recursive Store Ownership -**Phase status:** Not started +**Phase status:** Next ## Phase 11 — List and Search Ownership diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index e82c7f29..bae364ea 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -354,42 +354,68 @@ func TestCatalogContractSnapshotGraphAcrossBackends(t *testing.T) { }) } -// CAT-007 preserves the remaining deliberately deferred API boundary and proves -// those methods cannot return partial results or mutate the catalog. -func TestCatalogContractDeferredMethodsAcrossBackends(t *testing.T) { +// CAT-007 proves deterministic current/snapshot GC roots on both backends. +func TestCatalogContractGCPlansAcrossBackends(t *testing.T) { forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { seedCatalogFixture(t, backend.DB) svc := catalog.NewServiceFromSQL(backend.DB) before := catalogStateCounts(t, backend.DB) - gcPlan, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{}) - assertDeferred(t, "LoadGCPlanMetadata", err, gcPlan) + first, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{}) + if err != nil { + t.Fatalf("LoadGCPlanMetadata: %v", err) + } + second, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{}) + if err != nil || !reflect.DeepEqual(first, second) { + t.Fatalf("GC plan not deterministic: first=%+v second=%+v err=%v", first, second, err) + } + wantSnapshots := []string{"snap-full", "snap-tie-a", "snap-tie-b", "snap-child", "snap-null-label"} + gotSnapshots := make([]string, len(first.ProtectedSnapshots)) + for i, snapshot := range first.ProtectedSnapshots { + gotSnapshots[i] = snapshot.ID + } + if !reflect.DeepEqual(gotSnapshots, wantSnapshots) { + t.Fatalf("protected snapshots=%v want=%v", gotSnapshots, wantSnapshots) + } + wantRoots := []catalog.GCReachabilityRoot{ + {LogicalFileID: 1, Current: true, SnapshotIDs: []string{}}, + {LogicalFileID: 2, SnapshotIDs: []string{"snap-full", "snap-tie-a"}}, + {LogicalFileID: 3, Current: true, SnapshotIDs: []string{"snap-child"}}, + {LogicalFileID: 5, Current: true, SnapshotIDs: []string{}}, + } + if !reflect.DeepEqual(first.Roots, wantRoots) { + t.Fatalf("GC roots=%+v want=%+v", first.Roots, wantRoots) + } + excluded, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{ExcludeSnapshotIDs: []string{"snap-full", "snap-full"}}) + if err != nil { + t.Fatalf("excluded GC plan: %v", err) + } + if containsSnapshot(excluded.ProtectedSnapshots, "snap-full") || !reflect.DeepEqual(excluded.Roots[1].SnapshotIDs, []string{"snap-tie-a"}) { + t.Fatalf("excluded GC plan retained snap-full: %+v", excluded) + } + if plan, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{ExcludeSnapshotIDs: []string{"missing"}}); plan != nil || !catalog.IsCode(err, catalog.ErrorNotFound) { + t.Fatalf("missing excluded snapshot: plan=%+v err=%v", plan, err) + } + if plan, err := svc.LoadGCPlanMetadata(context.Background(), catalog.GCPlanInput{ExcludeSnapshotIDs: []string{" "}}); plan != nil || !catalog.IsCode(err, catalog.ErrorInvalidArgument) { + t.Fatalf("empty excluded snapshot: plan=%+v err=%v", plan, err) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if plan, err := svc.LoadGCPlanMetadata(cancelled, catalog.GCPlanInput{}); plan != nil || !catalog.IsCode(err, catalog.ErrorCancelled) || !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled GC plan: plan=%+v err=%v", plan, err) + } if after := catalogStateCounts(t, backend.DB); after != before { - t.Fatalf("deferred catalog methods mutated catalog state: before=%+v after=%+v", before, after) + t.Fatalf("GC plan read mutated catalog state: before=%+v after=%+v", before, after) } }) } -func assertDeferred(t *testing.T, name string, err error, result any) { - t.Helper() - if !errors.Is(err, catalog.ErrNotImplemented) || !catalog.IsDeferred(err) { - t.Errorf("%s: want catalog.ErrNotImplemented, got %v", name, err) - } - if !isNil(result) { - t.Errorf("%s: want nil result, got %#v", name, result) - } -} - -func isNil(value any) bool { - if value == nil { - return true - } - v := reflect.ValueOf(value) - switch v.Kind() { - case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Interface: - return v.IsNil() - default: - return false +func containsSnapshot(snapshots []catalog.SnapshotRef, id string) bool { + for _, snapshot := range snapshots { + if snapshot.ID == id { + return true + } } + return false } // CAT-007 adds bounded portable cancelled-context assertions. Errors must not diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index c7b34291..367f2a8a 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -79,8 +79,7 @@ type SnapshotCatalog interface { // SnapshotGraphCatalog exposes the snapshot lineage graph. // -// Phase 5 implements this contract. Until then it returns the typed -// ErrNotImplemented sentinel and no partial graph. +// Phase 5 implemented this contract. // Empty catalogs return empty ordered slices. Historical missing parents are // represented by SnapshotParentMissing; malformed cycles return a typed // invariant_violation. The catalog never invents or silently repairs edges. @@ -97,8 +96,8 @@ type ReachabilityCatalog interface { // PlacementCatalog exposes chunk/block/container placement metadata. // -// Phase 6 implements this contract. It represents packed and legacy roots as a -// strict tagged union. Until then it returns no partial placement result. +// Phase 6 implemented this contract. It represents packed and legacy roots as +// a strict tagged union and never returns a partial placement result. // A missing logical file returns not_found. A zero-length logical file returns // an empty placement slice. Missing, duplicate, mixed, or malformed placement // rows return invariant_violation rather than a partial recipe. @@ -108,9 +107,8 @@ type PlacementCatalog interface { // RestorePlanCatalog exposes restore-plan metadata. // -// Phase 7 implements this contract, where the "restore must not write +// Phase 7 implemented this contract, where the "restore must not write // outside destination" invariant is enforced at the engine/catalog boundary. -// Until then it returns no partial restore plan. // The service never opens or commits a transaction: constructing it with the // caller's *sql.Tx keeps selector resolution and recipe loading in that exact // transaction. @@ -124,8 +122,7 @@ type RestorePlanCatalog interface { // GCPlanCatalog exposes GC-plan metadata. // // Phase 9 implements and adopts this contract, where the "GC must never delete -// reachable data" invariant is tested at the engine/catalog boundary. Until -// then it returns no partial GC plan. +// reachable data" invariant is tested at the engine/catalog boundary. // Excluded snapshot IDs are validated before reads; missing IDs return a typed // not_found error, while malformed graph rows return invariant_violation. type GCPlanCatalog interface { diff --git a/internal/catalog/errors.go b/internal/catalog/errors.go index ad498f3a..39a15a65 100644 --- a/internal/catalog/errors.go +++ b/internal/catalog/errors.go @@ -69,8 +69,8 @@ func IsCode(err error, code ErrorCode) bool { var deferredCause = errors.New("catalog operation not implemented") -// ErrNotImplemented is the typed transitional sentinel for the four planning -// methods. Production must not consume a method while it returns this value. +// ErrNotImplemented is retained for error-taxonomy compatibility. No active +// catalog operation returns it after Phase 9 completed all four planning APIs. var ErrNotImplemented error = NewError( ErrorUnsupported, "deferred planning operation", diff --git a/internal/catalog/gc_plan.go b/internal/catalog/gc_plan.go index e2c02313..ca37a317 100644 --- a/internal/catalog/gc_plan.go +++ b/internal/catalog/gc_plan.go @@ -1,11 +1,138 @@ package catalog -import "context" +import ( + "context" + "errors" + "fmt" + "sort" +) // LoadGCPlanMetadata implements GCPlanCatalog. // // Phase 9 implements and adopts deterministic reachability metadata. func (s *Service) LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*GCPlanMetadata, error) { - _, _ = ctx, input - return nil, ErrNotImplemented + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, gcCatalogError(err) + } + normalized, err := NormalizeGCPlanInput(input) + if err != nil { + return nil, err + } + excluded := make(map[string]struct{}, len(normalized.ExcludeSnapshotIDs)) + for _, id := range normalized.ExcludeSnapshotIDs { + var exists bool + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM snapshot WHERE id = $1)`, id).Scan(&exists); err != nil { + return nil, gcCatalogError(fmt.Errorf("validate excluded snapshot %q: %w", id, err)) + } + if !exists { + return nil, NewError(ErrorNotFound, "load GC plan", "excluded_snapshot_exists", fmt.Sprintf("snapshot %q does not exist", id), nil) + } + excluded[id] = struct{}{} + } + + snapshots, err := s.ListSnapshots(ctx, SnapshotFilter{}) + if err != nil { + return nil, gcCatalogError(fmt.Errorf("load protected snapshots: %w", err)) + } + protected := make([]SnapshotRef, 0, len(snapshots)) + for _, snapshot := range snapshots { + if _, skip := excluded[snapshot.ID]; !skip { + protected = append(protected, snapshot) + } + } + sort.Slice(protected, func(i, j int) bool { + if !protected[i].CreatedAt.Equal(protected[j].CreatedAt) { + return protected[i].CreatedAt.Before(protected[j].CreatedAt) + } + return protected[i].ID < protected[j].ID + }) + snapshotRank := make(map[string]int, len(protected)) + for i, snapshot := range protected { + snapshotRank[snapshot.ID] = i + } + + roots := make(map[int64]*GCReachabilityRoot) + currentRows, err := s.db.QueryContext(ctx, `SELECT DISTINCT logical_file_id FROM physical_file ORDER BY logical_file_id`) + if err != nil { + return nil, gcCatalogError(fmt.Errorf("load current GC roots: %w", err)) + } + for currentRows.Next() { + var id int64 + if err := currentRows.Scan(&id); err != nil { + _ = currentRows.Close() + return nil, gcCatalogError(fmt.Errorf("scan current GC root: %w", err)) + } + root := ensureGCRoot(roots, id) + root.Current = true + } + if err := currentRows.Err(); err != nil { + _ = currentRows.Close() + return nil, gcCatalogError(fmt.Errorf("iterate current GC roots: %w", err)) + } + if err := currentRows.Close(); err != nil { + return nil, gcCatalogError(fmt.Errorf("close current GC roots: %w", err)) + } + + snapshotRows, err := s.db.QueryContext(ctx, `SELECT DISTINCT snapshot_id, logical_file_id FROM snapshot_file ORDER BY logical_file_id, snapshot_id`) + if err != nil { + return nil, gcCatalogError(fmt.Errorf("load snapshot GC roots: %w", err)) + } + for snapshotRows.Next() { + var snapshotID string + var logicalID int64 + if err := snapshotRows.Scan(&snapshotID, &logicalID); err != nil { + _ = snapshotRows.Close() + return nil, gcCatalogError(fmt.Errorf("scan snapshot GC root: %w", err)) + } + if _, skip := excluded[snapshotID]; skip { + continue + } + if _, known := snapshotRank[snapshotID]; !known { + _ = snapshotRows.Close() + return nil, NewError(ErrorInvariantViolation, "load GC plan", "snapshot_root_has_snapshot", fmt.Sprintf("snapshot root %q has no protected snapshot metadata", snapshotID), nil) + } + root := ensureGCRoot(roots, logicalID) + root.SnapshotIDs = append(root.SnapshotIDs, snapshotID) + } + if err := snapshotRows.Err(); err != nil { + _ = snapshotRows.Close() + return nil, gcCatalogError(fmt.Errorf("iterate snapshot GC roots: %w", err)) + } + if err := snapshotRows.Close(); err != nil { + return nil, gcCatalogError(fmt.Errorf("close snapshot GC roots: %w", err)) + } + + logicalIDs := make([]int64, 0, len(roots)) + for id := range roots { + logicalIDs = append(logicalIDs, id) + } + sort.Slice(logicalIDs, func(i, j int) bool { return logicalIDs[i] < logicalIDs[j] }) + orderedRoots := make([]GCReachabilityRoot, 0, len(logicalIDs)) + for _, id := range logicalIDs { + root := roots[id] + sort.SliceStable(root.SnapshotIDs, func(i, j int) bool { + return snapshotRank[root.SnapshotIDs[i]] < snapshotRank[root.SnapshotIDs[j]] + }) + orderedRoots = append(orderedRoots, *root) + } + return &GCPlanMetadata{Roots: orderedRoots, ProtectedSnapshots: protected}, nil +} + +func ensureGCRoot(roots map[int64]*GCReachabilityRoot, id int64) *GCReachabilityRoot { + if root := roots[id]; root != nil { + return root + } + root := &GCReachabilityRoot{LogicalFileID: id, SnapshotIDs: make([]string, 0)} + roots[id] = root + return root +} + +func gcCatalogError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return NewError(ErrorCancelled, "load GC plan", "", "GC plan load cancelled", err) + } + return NewError(ErrorOperationFailed, "load GC plan", "", "GC plan query failed", err) } diff --git a/internal/catalog/service_test.go b/internal/catalog/service_test.go index da601b58..9dc34d2f 100644 --- a/internal/catalog/service_test.go +++ b/internal/catalog/service_test.go @@ -3,7 +3,6 @@ package catalog_test import ( "context" "database/sql" - "errors" "testing" "time" @@ -389,28 +388,23 @@ func TestServiceLoadReachabilityRoots(t *testing.T) { assertReachabilityRoots(t, roots) } -// TestServiceDeferredMethodsReturnErrNotImplemented verifies the remaining GC -// skeleton method returns ErrNotImplemented. -func TestServiceDeferredMethodsReturnErrNotImplemented(t *testing.T) { +func TestServiceLoadGCPlanMetadataEmptyCatalog(t *testing.T) { dbconn := openTestDB(t) svc := catalog.NewServiceFromSQL(dbconn) ctx := context.Background() before := countCatalogLogicalFiles(t, dbconn) gcPlan, err := svc.LoadGCPlanMetadata(ctx, catalog.GCPlanInput{}) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Errorf("LoadGCPlanMetadata: want ErrNotImplemented via errors.Is, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Errorf("LoadGCPlanMetadata: want catalog.IsDeferred=true, got %v", err) + if err != nil { + t.Fatalf("LoadGCPlanMetadata: %v", err) } - if gcPlan != nil { - t.Errorf("LoadGCPlanMetadata: want nil metadata on deferred path, got %+v", gcPlan) + if gcPlan == nil || gcPlan.Roots == nil || gcPlan.ProtectedSnapshots == nil || len(gcPlan.Roots) != 0 || len(gcPlan.ProtectedSnapshots) != 0 { + t.Fatalf("LoadGCPlanMetadata: want non-nil empty slices, got %+v", gcPlan) } after := countCatalogLogicalFiles(t, dbconn) if after != before { - t.Fatalf("deferred catalog methods should not mutate logical_file rows: before=%d after=%d", before, after) + t.Fatalf("GC plan read should not mutate logical_file rows: before=%d after=%d", before, after) } } diff --git a/internal/gc/plan.go b/internal/gc/plan.go index 377f7b39..67e0aedc 100644 --- a/internal/gc/plan.go +++ b/internal/gc/plan.go @@ -13,6 +13,7 @@ import ( "sort" "strings" + "github.com/franchoy/coldkeep/internal/catalog" chunkmeta "github.com/franchoy/coldkeep/internal/chunk" "github.com/franchoy/coldkeep/internal/graph" "github.com/franchoy/coldkeep/internal/invariants" @@ -126,18 +127,21 @@ func BuildPlan(ctx context.Context, dbconn *sql.DB, opts PlanOptions) (*Plan, er if dbconn == nil { return nil, fmt.Errorf("gc.BuildPlan: nil db") } - if err := validateAssumeDeletedSnapshots(ctx, dbconn, opts.AssumeDeletedSnapshots); err != nil { - return nil, fmt.Errorf("gc.BuildPlan: validate assumed-deleted snapshots: %w", err) + metadata, err := catalog.NewServiceFromSQL(dbconn).LoadGCPlanMetadata(ctx, catalog.GCPlanInput{ExcludeSnapshotIDs: opts.AssumeDeletedSnapshots}) + if err != nil { + if catalog.IsCode(err, catalog.ErrorInvalidArgument) || catalog.IsCode(err, catalog.ErrorNotFound) { + return nil, fmt.Errorf("gc.BuildPlan: validate assumed-deleted snapshots: %w", err) + } + return nil, fmt.Errorf("gc.BuildPlan: gc roots: %w", err) } if err := refuseOnIntegrityIssues(dbconn); err != nil { return nil, fmt.Errorf("gc.BuildPlan: %w", err) } g := graph.NewService(dbconn) - - roots, err := g.GCRoots(ctx, graph.GCRootOptions{ExcludeSnapshots: opts.AssumeDeletedSnapshots}) - if err != nil { - return nil, fmt.Errorf("gc.BuildPlan: gc roots: %w", err) + roots := make([]graph.NodeID, 0, len(metadata.Roots)) + for _, root := range metadata.Roots { + roots = append(roots, graph.NodeID{Type: graph.EntityLogicalFile, ID: root.LogicalFileID}) } reachableChunkIDs, err := g.ReachableChunksFromRoots(ctx, roots) @@ -187,33 +191,6 @@ func BuildPlan(ctx context.Context, dbconn *sql.DB, opts PlanOptions) (*Plan, er return plan, nil } -func validateAssumeDeletedSnapshots(ctx context.Context, dbconn *sql.DB, snapshotIDs []string) error { - if len(snapshotIDs) == 0 { - return nil - } - - seen := make(map[string]struct{}, len(snapshotIDs)) - for _, snapshotID := range snapshotIDs { - if snapshotID == "" { - return fmt.Errorf("snapshot id must not be empty") - } - if _, exists := seen[snapshotID]; exists { - continue - } - seen[snapshotID] = struct{}{} - - var exists bool - if err := dbconn.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM snapshot WHERE id = $1)`, snapshotID).Scan(&exists); err != nil { - return err - } - if !exists { - return fmt.Errorf("snapshot %q does not exist", snapshotID) - } - } - - return nil -} - func loadAllCompletedChunks(ctx context.Context, dbconn *sql.DB) ([]chunkRecord, error) { rows, err := dbconn.QueryContext(ctx, ` SELECT id, live_ref_count, pin_count, chunker_version diff --git a/internal/maintenance/gc.go b/internal/maintenance/gc.go index 803715a2..4fc5bf48 100644 --- a/internal/maintenance/gc.go +++ b/internal/maintenance/gc.go @@ -9,6 +9,7 @@ import ( "log" "strings" + "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/fsx" @@ -43,7 +44,7 @@ var gcPhysicalIntegrityCheck = func(dbconn *sql.DB) (verify.PhysicalFileIntegrit } var gcComputeReachability = func(ctx context.Context, dbconn *sql.DB) (*retention.ReachabilitySummary, error) { - return retention.ComputeReachabilitySummary(ctx, dbconn) + return computeGCReachabilityFromCatalog(ctx, dbconn) } var gcMarkReachableChunks = func(ctx context.Context, dbconn *sql.DB) (map[int64]struct{}, error) { @@ -577,14 +578,38 @@ func MarkReachableChunks(ctx context.Context, dbconn *sql.DB) (map[int64]struct{ } g := graph.NewService(dbconn) - roots, err := g.GCRoots(ctx, graph.GCRootOptions{}) + metadata, err := catalog.NewServiceFromSQL(dbconn).LoadGCPlanMetadata(ctx, catalog.GCPlanInput{}) if err != nil { return nil, err } + roots := make([]graph.NodeID, 0, len(metadata.Roots)) + for _, root := range metadata.Roots { + roots = append(roots, graph.NodeID{Type: graph.EntityLogicalFile, ID: root.LogicalFileID}) + } return g.ReachableChunksFromRoots(ctx, roots) } +func computeGCReachabilityFromCatalog(ctx context.Context, dbconn *sql.DB) (*retention.ReachabilitySummary, error) { + metadata, err := catalog.NewServiceFromSQL(dbconn).LoadGCPlanMetadata(ctx, catalog.GCPlanInput{}) + if err != nil { + return nil, err + } + current := make(map[int64]struct{}) + snapshot := make(map[int64]struct{}) + retained := make(map[int64]struct{}, len(metadata.Roots)) + for _, root := range metadata.Roots { + retained[root.LogicalFileID] = struct{}{} + if root.Current { + current[root.LogicalFileID] = struct{}{} + } + if len(root.SnapshotIDs) != 0 { + snapshot[root.LogicalFileID] = struct{}{} + } + } + return &retention.ReachabilitySummary{CurrentLogicalIDs: current, SnapshotLogicalIDs: snapshot, RetainedLogicalIDs: retained}, nil +} + // LoadLivePackedBlockIDs resolves live chunks (live_ref_count > 0 OR pin_count > 0) // to their packed storage block ids through chunk_block_refs. func LoadLivePackedBlockIDs(ctx context.Context, dbconn *sql.DB) (map[int64]struct{}, error) { diff --git a/internal/observability/simulate.go b/internal/observability/simulate.go index 3fad9efc..b31b8d2d 100644 --- a/internal/observability/simulate.go +++ b/internal/observability/simulate.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/gc" - "github.com/franchoy/coldkeep/internal/graph" ) const SimulationKindGC = "gc" @@ -38,10 +38,8 @@ func (s *Service) simulateGC(ctx context.Context, opts SimulationOptions) (*Simu rootMetadata := map[string]any{ "excluded_snapshots": len(opts.AssumeDeletedSnapshots), } - if s.graph != nil { - if roots, rootsErr := s.graph.GCRoots(ctx, graph.GCRootOptions{ExcludeSnapshots: opts.AssumeDeletedSnapshots}); rootsErr == nil { - rootMetadata["root_count"] = len(roots) - } + if roots, rootsErr := catalog.NewServiceFromSQL(s.db).LoadGCPlanMetadata(ctx, catalog.GCPlanInput{ExcludeSnapshotIDs: opts.AssumeDeletedSnapshots}); rootsErr == nil { + rootMetadata["root_count"] = len(roots.Roots) } emitTrace(opts.Trace, TraceEvent{ Step: "simulate.gc.roots.load", From c1fef7ea44032ecc830225f1d8e814c2daa6c282 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 13:51:24 +0200 Subject: [PATCH 11/27] Complete v1.13.12 folder store ownership --- cmd/coldkeep/main.go | 26 +++-- cmd/coldkeep/stored_path_engine_adapters.go | 8 ++ .../stored_path_engine_routing_test.go | 98 +++++++++++++++++++ docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...10-folder-and-recursive-store-ownership.md | 65 ++++++++++++ docs/release/v1.13/v1.13.12-scope.md | 7 +- .../v1.13/v1.13.12-validation-checklist.md | 19 +++- internal/engine/candidates.go | 46 ++++----- internal/engine/candidates_test.go | 2 +- internal/engine/contracts_test.go | 25 +++-- .../engine/default_engine_routing_helpers.go | 48 +++++++-- internal/engine/engine.go | 5 + internal/engine/errors_test.go | 26 ----- .../mutation_backend_contract_helpers_test.go | 5 - .../engine/mutation_backend_contract_test.go | 50 +++++++++- internal/engine/store_engine_test.go | 32 +++++- internal/engine/unsupported_modes_test.go | 16 ++- .../engine/validation_vs_unsupported_test.go | 55 ++++++++--- internal/storage/store.go | 12 ++- 19 files changed, 421 insertions(+), 130 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase10-folder-and-recursive-store-ownership.md diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 190e0af9..4416e2db 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -1784,26 +1784,24 @@ func runStoreFolderCommand(parsed parsedCommandLine, outputMode cliOutputMode) e opts.StoreFolderWorkers = workers } - sgctx, err := storage.LoadDefaultStorageContext() + sgctx, err := loadDefaultStorageContextPhase() if err != nil { return fmt.Errorf("load storage context: %w", err) } defer func() { _ = sgctx.Close() }() - if codecName == "" { - err = storage.StoreFolderWithStorageContextAndOptions(sgctx, path, opts) - } else { - if codecName == "plain" { - _, _ = fmt.Fprintln(os.Stderr, "WARNING: data would be stored without encryption") - } - - codec, parseErr := blocks.ParseCodec(codecName) - if parseErr != nil { - return parseErr - } - - err = storage.StoreFolderWithStorageContextAndCodecAndOptions(sgctx, path, codec, opts) + if codecName == "plain" { + _, _ = fmt.Fprintln(os.Stderr, "WARNING: data would be stored without encryption") + } + eng, err := newStoreFolderCommandEngine(sgctx) + if err != nil { + return err } + _, err = eng.StoreFolder(context.Background(), engine.StoreFolderRequest{ + SourcePath: path, + Codec: codecName, + Workers: opts.StoreFolderWorkers, + }) if err != nil { return err } diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index fbbcbcab..0eb92537 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -25,6 +25,14 @@ var newSnapshotRestoreCommandEngine = func(sgctx storage.StorageContext) (engine }) } +var newStoreFolderCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{ + DB: sgctx.DB, + ContainerDir: sgctx.EffectiveContainerDir(), + StoreContext: &sgctx, + }) +} + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 157a3289..3339a23b 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -22,6 +22,7 @@ type stubCommandEngine struct { snapshotRestoreFunc func(context.Context, engine.SnapshotRestoreRequest) (engine.SnapshotRestoreResult, error) restoreStoredPathFunc func(context.Context, engine.RestoreStoredPathRequest) (engine.RestoreStoredPathResult, error) removeStoredPathsFunc func(context.Context, engine.RemoveStoredPathsRequest) (engine.RemoveStoredPathsResult, error) + storeFolderFunc func(context.Context, engine.StoreFolderRequest) (engine.StoreFolderResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -59,6 +60,85 @@ func (s stubCommandEngine) RemoveStoredPaths(ctx context.Context, req engine.Rem return engine.RemoveStoredPathsResult{}, errors.New("unexpected RemoveStoredPaths call") } +func (s stubCommandEngine) StoreFolder(ctx context.Context, req engine.StoreFolderRequest) (engine.StoreFolderResult, error) { + if s.storeFolderFunc != nil { + return s.storeFolderFunc(ctx, req) + } + return engine.StoreFolderResult{}, errors.New("unexpected StoreFolder call") +} + +func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ + storeFolderFunc: func(_ context.Context, req engine.StoreFolderRequest) (engine.StoreFolderResult, error) { + if req.SourcePath != "/input" || req.Codec != "aes-gcm" || req.Workers != 3 { + t.Fatalf("unexpected StoreFolder request: %+v", req) + } + return engine.StoreFolderResult{SourcePath: req.SourcePath, FilesStored: 2, BytesLogical: 11, WorkersUsed: 3}, nil + }, + }) + + output := captureStdout(t, func() { + err := runStoreFolderCommand(parsedCommandLine{ + method: "store-folder", + positionals: []string{"/input"}, + flags: map[string][]string{ + "codec": {"aes-gcm"}, + "workers": {"3"}, + }, + }, outputModeJSON) + if err != nil { + t.Fatalf("runStoreFolderCommand: %v", err) + } + }) + if strings.TrimSpace(output) != `{"command":"store-folder","status":"ok","target":"/input"}` { + t.Fatalf("unexpected JSON output: %s", output) + } +} + +func TestRunStoreFolderCommandUsesEngineTextParity(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ + storeFolderFunc: func(_ context.Context, req engine.StoreFolderRequest) (engine.StoreFolderResult, error) { + return engine.StoreFolderResult{SourcePath: req.SourcePath, FilesStored: 1}, nil + }, + }) + + output := captureStdout(t, func() { + err := runStoreFolderCommand(parsedCommandLine{ + method: "store-folder", + positionals: []string{"/input"}, + flags: map[string][]string{"codec": {"aes-gcm"}}, + }, outputModeText) + if err != nil { + t.Fatalf("runStoreFolderCommand: %v", err) + } + }) + for _, want := range []string{"Folder stored successfully: /input", "Hint: " + doctorOperationalHint} { + if !strings.Contains(output, want) { + t.Fatalf("expected output to contain %q, got:\n%s", want, output) + } + } +} + +func TestRunStoreFolderCommandPropagatesEngineErrors(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ + storeFolderFunc: func(context.Context, engine.StoreFolderRequest) (engine.StoreFolderResult, error) { + return engine.StoreFolderResult{}, errors.New("folder store failed") + }, + }) + + err := runStoreFolderCommand(parsedCommandLine{ + method: "store-folder", + positionals: []string{"/input"}, + flags: map[string][]string{"codec": {"aes-gcm"}}, + }, outputModeText) + if err == nil || err.Error() != "folder store failed" { + t.Fatalf("expected engine error to propagate unchanged, got %v", err) + } +} + func TestRunRestoreCommandStoredPathUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) restoreByIDCalled := false @@ -585,3 +665,21 @@ func installStoredPathCommandStubs( } } } + +func installStoreFolderCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { + t.Helper() + + originalLoad := loadDefaultStorageContextPhase + originalNewEngine := newStoreFolderCommandEngine + t.Cleanup(func() { + loadDefaultStorageContextPhase = originalLoad + newStoreFolderCommandEngine = originalNewEngine + }) + + loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { + return storage.StorageContext{DB: dbconn}, nil + } + newStoreFolderCommandEngine = func(storage.StorageContext) (engine.Engine, error) { + return stub, nil + } +} diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index e57131b7..0f369b60 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -69,11 +69,13 @@ ## Phase 10 — Folder and Recursive Store Ownership -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 10 folder store ownership](v1.13.12-phase10-folder-and-recursive-store-ownership.md). ## Phase 11 — List and Search Ownership -**Status:** Not started +**Status:** Next ## Phase 12 — Configuration Ownership diff --git a/docs/release/v1.13/v1.13.12-phase10-folder-and-recursive-store-ownership.md b/docs/release/v1.13/v1.13.12-phase10-folder-and-recursive-store-ownership.md new file mode 100644 index 00000000..3f10db66 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase10-folder-and-recursive-store-ownership.md @@ -0,0 +1,65 @@ +# Coldkeep v1.13.12 Phase 10 — Folder and Recursive Store Ownership + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `10 — Folder and Recursive Store Ownership` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Contract activation + +`Engine.StoreFolder` is now a distinct active operation. Its neutral request +contains the source path, optional codec override, and worker count; its result +contains source identity plus aggregate files, logical bytes, and effective +workers. A zero worker count selects the existing execution default. + +The single-file `StoreRequest` no longer advertises recursive or worker fields. +Dormant store-result fields that production never populated were also removed, +so both active contracts describe only supported behavior. + +## Production ownership + +The production `store-folder` command now loads its configured storage context, +constructs an injected engine, and invokes `Engine.StoreFolder`. The CLI retains +argument and environment ingestion, the existing unencrypted-data warning, and +the exact text/JSON projection. + +Behind the engine, the established storage pipeline still owns filesystem +discovery, normalized deterministic job construction, worker isolation, retry, +per-file store execution, aggregate statistics, cancellation, partial-error +return, and worker-writer finalization. The new context-aware entry point lets +engine cancellation reach that pipeline without changing storage or transaction +semantics. The `simulate store-folder` route remains the isolated simulation +tooling exclusion frozen in Phase 1. + +## Compatibility + +No CLI syntax, text, JSON, warning, exit-code, schema, backend default, +repository format, stored-path, transaction, container, codec, retry, or +finalization behavior changed. Explicit and default codecs continue through the +same block codec loaders, and the existing worker-selection policy remains +authoritative. + +## Validation + +- Direct engine folder-store, validation, and pre-cancellation tests: pass. +- CLI request translation, text/JSON parity, and error propagation: pass. +- Existing storage folder worker, aggregate, determinism, failure, and + concurrent-GC tests: pass. +- Shared SQLite/PostgreSQL engine folder-store contract: pass. +- Focused folder-store race suites across engine, storage, and CLI: pass. +- Full `go test -count=1 ./...`: pass. +- Validation root: `/tmp/coldkeep-v11312-phase10.t3lChX/`, containing isolated + caches, temporary files, storage roots, and SQLite databases. + +PostgreSQL validation created and dropped exactly +`coldkeep_bt_testenginestorefolderacrossbackends_postgr_2986e6_1` during the +initial fail-closed bootstrap check and +`coldkeep_bt_testenginestorefolderacrossbackends_postgr_29882b_1` during the +passing dual-backend run. A read-only maintenance query confirmed both exact +names are absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 11 is authorized to activate typed catalog and engine list/search +ownership without changing current-file query or presentation semantics. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index e6966d5b..14bac469 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,10 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–9 are Complete. Phase 10 is Next: add a distinct engine-owned folder -store operation while preserving traversal, deterministic scheduling, worker, -warning, aggregate-result, stored-path, partial-failure, and finalization -semantics. +Phases 0–10 are Complete. Phase 11 is Next: add typed catalog and engine +current-file list/search operations while preserving filtering, normalization, +sorting, pagination, JSON, text, and stored-path semantics. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 5da9f287..17156bd4 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -151,11 +151,26 @@ ## Phase 10 — Folder and Recursive Store Ownership -**Phase status:** Next +**Phase status:** Complete + +- [x] Single-file `StoreRequest` no longer advertises recursive or worker + fields and its result contains only implemented metadata. +- [x] `Engine.StoreFolder` owns the recursive operation boundary, codec + selection, worker policy, deterministic execution, aggregate result, and + cancellation propagation. +- [x] The existing storage pipeline retains traversal, per-worker isolation, + retry, partial failure, and writer-finalization semantics behind the engine. +- [x] Production `store-folder` CLI code performs only argument/environment + translation, warning/output projection, context loading, and engine dispatch. +- [x] Simulation remains the explicitly isolated tooling exclusion frozen in + Phase 1. +- [x] Direct, CLI-routing, cancellation, race, and SQLite/PostgreSQL contract + tests pass; exact PostgreSQL scratch databases were removed and confirmed + absent. ## Phase 11 — List and Search Ownership -**Phase status:** Not started +**Phase status:** Next ## Phase 12 — Configuration Ownership diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index 711cb1ab..a05ab380 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -114,26 +114,14 @@ type SnapshotQuery struct { // StoreRequest is the active request contract for Engine.Store. // -// Active semantics are single-file only. Recursive/folder-store ownership -// remains outside the active route, and Engine.Store returns ErrNotImplemented -// when Recursive is true. Any future contract split or narrowing requires an -// explicit v2.0 design. +// Store is intentionally single-file only. Folder traversal and aggregation +// use the distinct StoreFolder operation. type StoreRequest struct { - // SourcePath is the file or folder to store. + // SourcePath is the file to store. SourcePath string // Codec selects the storage codec (e.g. "plain", "aes-gcm"). Empty means // the repository default. Codec string - // Recursive requests folder store semantics (store-folder). - // Active Engine.Store callers must leave this false; true returns - // ErrNotImplemented. - Recursive bool - // Workers is the parallelism for folder store; zero means the default. - // This remains outside the active route until an explicit v2.0 contract - // decision addresses recursive folder-store ownership. - Workers int - // Tags carries optional caller-supplied tags. - Tags []string } // StoreResult is the active result contract for Engine.Store. @@ -144,21 +132,29 @@ type StoreResult struct { StoredPath string // LogicalFileID identifies the stored logical file. LogicalFileID int64 - // PhysicalFileID identifies the underlying physical file when applicable. - PhysicalFileID int64 // FileHash is the content hash (e.g. SHA-256) of the stored file. FileHash string // AlreadyStored indicates the content was already present (dedup hit). AlreadyStored bool - // BytesLogical is the logical (pre-transform) size in bytes. +} + +// StoreFolderRequest is the recursive folder-store contract. Workers zero +// selects the established default; positive values request bounded file-level +// fan-out subject to writer capability. +type StoreFolderRequest struct { + SourcePath string + Codec string + Workers int +} + +// StoreFolderResult reports deterministic aggregate execution statistics. +// Partial statistics are returned with an error when work fails after some +// files have completed. +type StoreFolderResult struct { + SourcePath string + FilesStored int BytesLogical int64 - // BytesStored is the physical (post-transform) size in bytes. - BytesStored int64 - // ChunksCreated and ChunksReused describe chunk-level dedup outcomes. - ChunksCreated int - ChunksReused int - // Warnings carries structured, non-fatal warnings. - Warnings []OperationWarning + WorkersUsed int } // --------------------------------------------------------------------------- diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index 7f5f99d8..f5a18ff0 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 361a58bd..b3b8af77 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -42,6 +42,8 @@ func activeCoreContractTypes() []struct { {"SnapshotQuery", engine.SnapshotQuery{}}, {"StoreRequest", engine.StoreRequest{}}, {"StoreResult", engine.StoreResult{}}, + {"StoreFolderRequest", engine.StoreFolderRequest{}}, + {"StoreFolderResult", engine.StoreFolderResult{}}, {"RestoreRequest", engine.RestoreRequest{}}, {"RestoreItemResult", engine.RestoreItemResult{}}, {"RestoreResult", engine.RestoreResult{}}, @@ -649,23 +651,28 @@ func assertRecoverRequestIsCorrectiveOnly(t *testing.T, req engine.RecoverReques } } -// TestStoreContractRepresentsFileAndFolder proves store covers single file and -// recursive folder store with codec and workers. -func TestStoreContractRepresentsFileAndFolder(t *testing.T) { +// TestStoreContractsSeparateFileAndFolder proves recursive traversal is a +// distinct operation rather than dormant fields on the single-file request. +func TestStoreContractsSeparateFileAndFolder(t *testing.T) { file := engine.StoreRequest{SourcePath: "f.txt", Codec: "aes-gcm"} - folder := engine.StoreRequest{SourcePath: "dir", Recursive: true, Workers: 8, Codec: "plain"} - if file.Recursive || !folder.Recursive || folder.Workers != 8 { + folder := engine.StoreFolderRequest{SourcePath: "dir", Workers: 8, Codec: "plain"} + if file.SourcePath != "f.txt" || folder.SourcePath != "dir" || folder.Workers != 8 { t.Fatalf("store contract not representable: file=%+v folder=%+v", file, folder) } - res := engine.StoreResult{ + fileResult := engine.StoreResult{ SourcePath: "f.txt", StoredPath: "f.txt", LogicalFileID: 1, FileHash: "h", AlreadyStored: true, - ChunksReused: 3, } - if !res.AlreadyStored || res.ChunksReused != 3 { - t.Fatalf("store result not representable: %+v", res) + folderResult := engine.StoreFolderResult{ + SourcePath: "dir", + FilesStored: 3, + BytesLogical: 42, + WorkersUsed: 2, + } + if !fileResult.AlreadyStored || folderResult.FilesStored != 3 || folderResult.BytesLogical != 42 { + t.Fatalf("store results not representable: file=%+v folder=%+v", fileResult, folderResult) } } diff --git a/internal/engine/default_engine_routing_helpers.go b/internal/engine/default_engine_routing_helpers.go index 2071be70..5a1d27a1 100644 --- a/internal/engine/default_engine_routing_helpers.go +++ b/internal/engine/default_engine_routing_helpers.go @@ -10,6 +10,7 @@ import ( "github.com/franchoy/coldkeep/internal/blocks" "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/execution" "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/maintenance" "github.com/franchoy/coldkeep/internal/snapshot" @@ -126,9 +127,6 @@ func (e *DefaultEngine) Store(ctx context.Context, req StoreRequest) (StoreResul if err := ctx.Err(); err != nil { return StoreResult{}, err } - if req.Recursive { - return StoreResult{}, ErrNotImplemented - } if strings.TrimSpace(req.SourcePath) == "" { return StoreResult{}, fmt.Errorf("engine: store source path is required") } @@ -141,15 +139,47 @@ func (e *DefaultEngine) Store(ctx context.Context, req StoreRequest) (StoreResul return StoreResult{}, err } return StoreResult{ - SourcePath: req.SourcePath, - StoredPath: stored.Path, - LogicalFileID: stored.FileID, - FileHash: stored.FileHash, - AlreadyStored: stored.AlreadyStored, - PhysicalFileID: 0, + SourcePath: req.SourcePath, + StoredPath: stored.Path, + LogicalFileID: stored.FileID, + FileHash: stored.FileHash, + AlreadyStored: stored.AlreadyStored, }, nil } +func (e *DefaultEngine) StoreFolder(ctx context.Context, req StoreFolderRequest) (StoreFolderResult, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return StoreFolderResult{}, err + } + if strings.TrimSpace(req.SourcePath) == "" { + return StoreFolderResult{}, fmt.Errorf("engine: store folder source path is required") + } + if req.Workers < 0 { + return StoreFolderResult{}, fmt.Errorf("engine: store folder workers must be zero or greater") + } + if e.config.StoreContext == nil { + return StoreFolderResult{}, fmt.Errorf("engine: store folder requires injected StoreContext") + } + workers := req.Workers + if workers == 0 { + workers = execution.DefaultOptions().StoreFolderWorkers + } + opts := execution.Options{StoreFolderWorkers: workers, PipelineDepth: 1, Deterministic: true} + codec, err := blocks.LoadDefaultCodec() + if strings.TrimSpace(req.Codec) != "" { + codec, err = blocks.ParseCodec(req.Codec) + } + if err != nil { + return StoreFolderResult{}, err + } + stats, err := storage.StoreFolderWithStorageContextAndCodecAndOptionsWithStatsContext(ctx, *e.config.StoreContext, req.SourcePath, codec, opts) + result := StoreFolderResult{SourcePath: req.SourcePath, FilesStored: stats.TotalFilesProcessed, BytesLogical: stats.TotalBytesProcessed, WorkersUsed: stats.WorkersUsed} + return result, err +} + func storeWithOptionalCodec(ctx storage.StorageContext, req StoreRequest) (storage.StoreFileResult, error) { if strings.TrimSpace(req.Codec) == "" { return storage.StoreFileWithStorageContextResult(ctx, req.SourcePath) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 5b90d9f0..7dd51aef 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -79,6 +79,11 @@ type Engine interface { // Active semantics are limited to single-file store. Store(ctx context.Context, req StoreRequest) (StoreResult, error) + // StoreFolder recursively stores a directory with deterministic discovery, + // bounded workers, aggregate results, and worker-writer finalization owned + // below this engine boundary. + StoreFolder(ctx context.Context, req StoreFolderRequest) (StoreFolderResult, error) + // Remove removes logical files from the repository by logical file ID. // Safety invariant: Remove must never make valid data unrecoverable. // Method selection owns addressing semantics: this method is by-ID only. diff --git a/internal/engine/errors_test.go b/internal/engine/errors_test.go index a6774423..4f035d84 100644 --- a/internal/engine/errors_test.go +++ b/internal/engine/errors_test.go @@ -6,10 +6,8 @@ import ( "fmt" "testing" - "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/invariants" - "github.com/franchoy/coldkeep/internal/storage" ) func TestErrNotImplementedRemainsUnsupportedSentinel(t *testing.T) { @@ -44,30 +42,6 @@ func TestIsUnsupportedRejectsUnrelatedErrors(t *testing.T) { } } -func TestUnsupportedEngineModesRemainRecognizedByIsUnsupported(t *testing.T) { - t.Run("recursive store", func(t *testing.T) { - db := openSnapshotTestDB(t) - sgctx := storage.StorageContext{ - DB: db, - Writer: container.NewSimulatedWriter(1024 * 1024), - ContainerDir: t.TempDir(), - } - eng, err := engine.New(engine.Config{DB: db, ContainerDir: sgctx.ContainerDir, StoreContext: &sgctx}) - if err != nil { - t.Fatalf("engine.New: %v", err) - } - _, err = eng.Store(context.Background(), engine.StoreRequest{ - SourcePath: t.TempDir(), - Recursive: true, - Workers: 2, - Codec: "plain", - }) - if !engine.IsUnsupported(err) { - t.Fatalf("expected recursive store error to classify as unsupported, got %v", err) - } - }) -} - func TestEngineErrorCodesAreStable(t *testing.T) { want := map[engine.ErrorCode]string{ engine.ErrorInvalidArgument: "invalid_argument", diff --git a/internal/engine/mutation_backend_contract_helpers_test.go b/internal/engine/mutation_backend_contract_helpers_test.go index 79df4556..3d73ac05 100644 --- a/internal/engine/mutation_backend_contract_helpers_test.go +++ b/internal/engine/mutation_backend_contract_helpers_test.go @@ -440,11 +440,6 @@ func assertStoreResultShape(t *testing.T, result engine.StoreResult) { result.LogicalFileID <= 0 || result.FileHash == "" { t.Fatalf("Store returned incomplete stable fields: %+v", result) } - if result.PhysicalFileID != 0 || result.BytesLogical != 0 || - result.BytesStored != 0 || result.ChunksCreated != 0 || - result.ChunksReused != 0 || result.Warnings != nil { - t.Fatalf("Store dormant result fields changed: %+v", result) - } } func seedMutationDeadContainer( diff --git a/internal/engine/mutation_backend_contract_test.go b/internal/engine/mutation_backend_contract_test.go index 9e818293..3d18c970 100644 --- a/internal/engine/mutation_backend_contract_test.go +++ b/internal/engine/mutation_backend_contract_test.go @@ -14,6 +14,41 @@ import ( "github.com/franchoy/coldkeep/internal/testutil/backendtest" ) +func TestEngineStoreFolderAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + fixture := newMutationBackendFixture(t, backend) + folder := filepath.Join(fixture.inputRoot, "folder") + for relativePath, payload := range map[string][]byte{ + "a.txt": []byte("alpha"), + "nested/b.txt": []byte("bravo"), + "nested/c.txt": []byte("charlie"), + } { + path := filepath.Join(folder, filepath.FromSlash(relativePath)) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("create folder input parent: %v", err) + } + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write folder input: %v", err) + } + } + + result, err := fixture.engine.StoreFolder(context.Background(), engine.StoreFolderRequest{ + SourcePath: folder, + Codec: "plain", + Workers: 2, + }) + if err != nil { + t.Fatalf("StoreFolder: %v", err) + } + if result.SourcePath != folder || result.FilesStored != 3 || result.BytesLogical != 17 || result.WorkersUsed != 2 { + t.Fatalf("unexpected StoreFolder result: %+v", result) + } + if got := mutationBackendInt64(t, backend.DB, `SELECT COUNT(*) FROM physical_file`); got != 3 { + t.Fatalf("physical file count: got %d want 3", got) + } + }) +} + func TestEngineMutationStoreRemoveAcrossBackends(t *testing.T) { backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { fixture := newMutationBackendFixture(t, backend) @@ -409,6 +444,7 @@ func TestEngineMutationErrorsAcrossBackends(t *testing.T) { cancelled, cancel := context.WithCancel(context.Background()) cancel() + cancelledFolder := t.TempDir() if _, err := fixture.engine.Store(cancelled, engine.StoreRequest{ SourcePath: "retained.txt", Codec: "plain", }); !errors.Is(err, context.Canceled) { @@ -418,6 +454,12 @@ func TestEngineMutationErrorsAcrossBackends(t *testing.T) { name string call func() error }{ + {"StoreFolder", func() error { + _, err := fixture.engine.StoreFolder(cancelled, engine.StoreFolderRequest{ + SourcePath: cancelledFolder, Codec: "plain", Workers: 2, + }) + return err + }}, {"Remove", func() error { _, err := eng.Remove(cancelled, engine.RemoveRequest{FileIDs: []int64{retained.LogicalFileID}}) return err @@ -467,10 +509,10 @@ func TestEngineMutationErrorsAcrossBackends(t *testing.T) { } }) } - if _, err := fixture.engine.Store(context.Background(), engine.StoreRequest{ - SourcePath: "retained.txt", Recursive: true, - }); !errors.Is(err, engine.ErrNotImplemented) || !engine.IsUnsupported(err) { - t.Fatalf("recursive Store classification: %v", err) + if _, err := fixture.engine.StoreFolder(context.Background(), engine.StoreFolderRequest{ + SourcePath: " ", Codec: "plain", + }); err == nil || engine.IsUnsupported(err) { + t.Fatalf("blank-path StoreFolder classification: %v", err) } if _, err := fixture.engine.Store(context.Background(), engine.StoreRequest{ SourcePath: " ", diff --git a/internal/engine/store_engine_test.go b/internal/engine/store_engine_test.go index 1e0ff97c..dfe6f2e0 100644 --- a/internal/engine/store_engine_test.go +++ b/internal/engine/store_engine_test.go @@ -2,7 +2,6 @@ package engine_test import ( "context" - "errors" "os" "path/filepath" "testing" @@ -45,7 +44,7 @@ func TestStoreByFileThroughEngine(t *testing.T) { } } -func TestStoreFolderDeferred(t *testing.T) { +func TestStoreFolderThroughEngine(t *testing.T) { db := openSnapshotTestDB(t) sgctx := storage.StorageContext{ DB: db, @@ -57,9 +56,32 @@ func TestStoreFolderDeferred(t *testing.T) { t.Fatalf("engine.New: %v", err) } - _, err = eng.Store(context.Background(), engine.StoreRequest{SourcePath: t.TempDir(), Recursive: true, Codec: "plain"}) - if !errors.Is(err, engine.ErrNotImplemented) { - t.Fatalf("expected ErrNotImplemented for recursive store, got %v", err) + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "nested"), 0o700); err != nil { + t.Fatalf("create nested directory: %v", err) + } + for path, payload := range map[string][]byte{ + filepath.Join(root, "a.txt"): []byte("alpha"), + filepath.Join(root, "nested", "b.txt"): []byte("bravo"), + } { + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write input %q: %v", path, err) + } + } + + res, err := eng.StoreFolder(context.Background(), engine.StoreFolderRequest{ + SourcePath: root, + Codec: "plain", + Workers: 4, + }) + if err != nil { + t.Fatalf("StoreFolder: %v", err) + } + if res.SourcePath != root || res.FilesStored != 2 || res.BytesLogical != 10 || res.WorkersUsed != 1 { + t.Fatalf("unexpected StoreFolder result: %+v", res) + } + if got := countLogicalFiles(t, db); got != 2 { + t.Fatalf("logical file count: got %d want 2", got) } } diff --git a/internal/engine/unsupported_modes_test.go b/internal/engine/unsupported_modes_test.go index 0cd4d19d..1033c3c0 100644 --- a/internal/engine/unsupported_modes_test.go +++ b/internal/engine/unsupported_modes_test.go @@ -11,7 +11,7 @@ import ( "github.com/franchoy/coldkeep/internal/storage" ) -func TestStoreRecursiveReturnsErrNotImplementedWithoutMutation(t *testing.T) { +func TestStoreFolderPreCancelledWithoutMutation(t *testing.T) { db := openSnapshotTestDB(t) sgctx := storage.StorageContext{ DB: db, @@ -24,21 +24,19 @@ func TestStoreRecursiveReturnsErrNotImplementedWithoutMutation(t *testing.T) { } before := countLogicalFiles(t, db) - _, err = eng.Store(context.Background(), engine.StoreRequest{ + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = eng.StoreFolder(ctx, engine.StoreFolderRequest{ SourcePath: t.TempDir(), - Recursive: true, Workers: 4, Codec: "plain", }) - if !errors.Is(err, engine.ErrNotImplemented) { - t.Fatalf("expected ErrNotImplemented for recursive store, got %v", err) - } - if !engine.IsUnsupported(err) { - t.Fatalf("expected recursive store error to classify as unsupported, got %v", err) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context cancellation, got %v", err) } after := countLogicalFiles(t, db) if after != before { - t.Fatalf("recursive unsupported mode should not mutate logical_file rows: before=%d after=%d", before, after) + t.Fatalf("cancelled folder store should not mutate logical_file rows: before=%d after=%d", before, after) } } diff --git a/internal/engine/validation_vs_unsupported_test.go b/internal/engine/validation_vs_unsupported_test.go index 13883e55..a7317f7d 100644 --- a/internal/engine/validation_vs_unsupported_test.go +++ b/internal/engine/validation_vs_unsupported_test.go @@ -10,8 +10,8 @@ import ( "github.com/franchoy/coldkeep/internal/storage" ) -func TestUnsupportedEngineModesRemainUnsupportedBoundaries(t *testing.T) { - t.Run("recursive store", func(t *testing.T) { +func TestValidationErrorsRemainOutsideUnsupportedClassification(t *testing.T) { + t.Run("store requires source path", func(t *testing.T) { db := openSnapshotTestDB(t) sgctx := storage.StorageContext{ DB: db, @@ -23,18 +23,11 @@ func TestUnsupportedEngineModesRemainUnsupportedBoundaries(t *testing.T) { t.Fatalf("engine.New: %v", err) } - _, err = eng.Store(context.Background(), engine.StoreRequest{ - SourcePath: t.TempDir(), - Recursive: true, - Workers: 2, - Codec: "plain", - }) - assertUnsupportedBoundary(t, err, engine.ErrNotImplemented.Error()) + _, err = eng.Store(context.Background(), engine.StoreRequest{SourcePath: "", Codec: "plain"}) + assertValidationBoundary(t, err, "engine: store source path is required") }) -} -func TestValidationErrorsRemainOutsideUnsupportedClassification(t *testing.T) { - t.Run("store requires source path", func(t *testing.T) { + t.Run("store folder requires source path", func(t *testing.T) { db := openSnapshotTestDB(t) sgctx := storage.StorageContext{ DB: db, @@ -46,8 +39,42 @@ func TestValidationErrorsRemainOutsideUnsupportedClassification(t *testing.T) { t.Fatalf("engine.New: %v", err) } - _, err = eng.Store(context.Background(), engine.StoreRequest{SourcePath: "", Codec: "plain"}) - assertValidationBoundary(t, err, "engine: store source path is required") + _, err = eng.StoreFolder(context.Background(), engine.StoreFolderRequest{Codec: "plain"}) + assertValidationBoundary(t, err, "engine: store folder source path is required") + }) + + t.Run("store folder rejects negative workers", func(t *testing.T) { + db := openSnapshotTestDB(t) + sgctx := storage.StorageContext{ + DB: db, + Writer: container.NewSimulatedWriter(1024 * 1024), + ContainerDir: t.TempDir(), + } + eng, err := engine.New(engine.Config{DB: db, ContainerDir: sgctx.ContainerDir, StoreContext: &sgctx}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + + _, err = eng.StoreFolder(context.Background(), engine.StoreFolderRequest{ + SourcePath: t.TempDir(), + Codec: "plain", + Workers: -1, + }) + assertValidationBoundary(t, err, "engine: store folder workers must be zero or greater") + }) + + t.Run("store folder requires injected StoreContext", func(t *testing.T) { + db := openSnapshotTestDB(t) + eng, err := engine.New(engine.Config{DB: db, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + + _, err = eng.StoreFolder(context.Background(), engine.StoreFolderRequest{ + SourcePath: t.TempDir(), + Codec: "plain", + }) + assertValidationBoundary(t, err, "engine: store folder requires injected StoreContext") }) t.Run("store requires injected StoreContext", func(t *testing.T) { diff --git a/internal/storage/store.go b/internal/storage/store.go index 302e1f18..ceedd15d 100644 --- a/internal/storage/store.go +++ b/internal/storage/store.go @@ -3097,6 +3097,16 @@ func StoreFolderWithStorageContextAndCodecAndOptions(sgctx StorageContext, root } func StoreFolderWithStorageContextAndCodecAndOptionsWithStats(sgctx StorageContext, root string, codec blocks.Codec, opts execution.Options) (execution.ExecutionStats, error) { + return StoreFolderWithStorageContextAndCodecAndOptionsWithStatsContext(context.Background(), sgctx, root, codec, opts) +} + +func StoreFolderWithStorageContextAndCodecAndOptionsWithStatsContext(ctx context.Context, sgctx StorageContext, root string, codec blocks.Codec, opts execution.Options) (execution.ExecutionStats, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return execution.ExecutionStats{}, err + } // Default to a single worker for deterministic append ordering and safer // container mutation semantics under mixed file sizes. err := opts.Validate() @@ -3119,7 +3129,7 @@ func StoreFolderWithStorageContextAndCodecAndOptionsWithStats(sgctx StorageConte return execution.ExecutionStats{}, err } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) defer cancel() jobCh := make(chan FileJob, 256) From 27f3b03e0e5ae3b38667b1a42de3e8fb574d8490 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 14:03:01 +0200 Subject: [PATCH 12/27] Complete v1.13.12 list and search ownership --- .../current_files_engine_routing_test.go | 94 ++++++++++ cmd/coldkeep/main.go | 143 +++++++++------- cmd/coldkeep/main_test.go | 31 ++-- cmd/coldkeep/stored_path_engine_adapters.go | 3 + .../stored_path_engine_routing_test.go | 16 ++ docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...13.12-phase11-list-and-search-ownership.md | 64 +++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 16 +- internal/catalog/backend_contract_test.go | 56 ++++++ internal/catalog/catalog.go | 7 + internal/catalog/current_files.go | 137 +++++++++++++++ internal/catalog/service.go | 1 + internal/catalog/types.go | 26 +++ internal/engine/candidates.go | 35 ++++ internal/engine/candidates_test.go | 2 +- internal/engine/contracts_test.go | 5 + internal/engine/current_files.go | 88 ++++++++++ internal/engine/current_files_test.go | 83 +++++++++ internal/engine/engine.go | 6 + internal/listing/list.go | 74 -------- internal/listing/list_test.go | 137 --------------- internal/listing/query.go | 68 -------- internal/listing/query_test.go | 133 -------------- internal/listing/search.go | 112 ------------ internal/listing/search_test.go | 162 ------------------ tests/integration/integration_test.go | 18 +- 27 files changed, 757 insertions(+), 772 deletions(-) create mode 100644 cmd/coldkeep/current_files_engine_routing_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase11-list-and-search-ownership.md create mode 100644 internal/catalog/current_files.go create mode 100644 internal/engine/current_files.go create mode 100644 internal/engine/current_files_test.go delete mode 100644 internal/listing/list.go delete mode 100644 internal/listing/list_test.go delete mode 100644 internal/listing/query.go delete mode 100644 internal/listing/query_test.go delete mode 100644 internal/listing/search.go delete mode 100644 internal/listing/search_test.go diff --git a/cmd/coldkeep/current_files_engine_routing_test.go b/cmd/coldkeep/current_files_engine_routing_test.go new file mode 100644 index 00000000..5ca95362 --- /dev/null +++ b/cmd/coldkeep/current_files_engine_routing_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" +) + +func TestRunListCommandUsesEngineJSONParity(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installCurrentFileCommandStubs(t, dbconn, stubCommandEngine{ + listFilesFunc: func(_ context.Context, req engine.ListFilesRequest) (engine.ListFilesResult, error) { + if req.Limit == nil || *req.Limit != 2 || req.Offset == nil || *req.Offset != 1 { + t.Fatalf("unexpected ListFiles request: %+v", req) + } + return engine.ListFilesResult{Files: []engine.CurrentFile{{ + ID: 7, Name: "/docs/a.txt", FileHash: "abc", SizeBytes: 12, CreatedAt: "2026-01-02 03:04:05", + }}}, nil + }, + }) + + output := captureStdout(t, func() { + err := runListCommand(parsedCommandLine{ + method: "list", flags: map[string][]string{"limit": {"2"}, "offset": {"1"}}, + }, outputModeJSON) + if err != nil { + t.Fatalf("runListCommand: %v", err) + } + }) + want := `{"command":"list","files":[{"id":7,"name":"/docs/a.txt","file_hash":"abc","size_bytes":12,"created_at":"2026-01-02 03:04:05"}],"status":"ok"}` + if strings.TrimSpace(output) != want { + t.Fatalf("unexpected list JSON:\n%s", output) + } +} + +func TestRunSearchCommandUsesTypedEngineFiltersAndTextParity(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installCurrentFileCommandStubs(t, dbconn, stubCommandEngine{ + searchFilesFunc: func(_ context.Context, req engine.SearchFilesRequest) (engine.SearchFilesResult, error) { + if strings.Join(req.NameContains, ",") != "docs,report" || len(req.MinSizeBytes) != 2 || req.MinSizeBytes[0] != 10 || req.MinSizeBytes[1] != 20 || len(req.MaxSizeBytes) != 1 || req.MaxSizeBytes[0] != 100 { + t.Fatalf("unexpected SearchFiles request: %+v", req) + } + return engine.SearchFilesResult{Files: []engine.CurrentFile{{ + ID: 9, Name: "/docs/report.txt", SizeBytes: 42, CreatedAt: "2026-02-03 04:05:06", + }}}, nil + }, + }) + + output := captureStdout(t, func() { + err := runSearchCommand(parsedCommandLine{ + method: "search", + flags: map[string][]string{ + "name": {"docs", "report"}, "min-size": {"10", "20"}, "max-size": {"100"}, + }, + }, outputModeText) + if err != nil { + t.Fatalf("runSearchCommand: %v", err) + } + }) + for _, want := range []string{"ID", "PATH", "/docs/report.txt", "42", "2026-02-03 04:05:06"} { + if !strings.Contains(output, want) { + t.Fatalf("expected search output to contain %q, got:\n%s", want, output) + } + } +} + +func TestRunCurrentFileCommandsPropagateEngineErrors(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installCurrentFileCommandStubs(t, dbconn, stubCommandEngine{ + listFilesFunc: func(context.Context, engine.ListFilesRequest) (engine.ListFilesResult, error) { + return engine.ListFilesResult{}, errors.New("list failed") + }, + }) + err := runListCommand(parsedCommandLine{method: "list", flags: map[string][]string{}}, outputModeText) + if err == nil || err.Error() != "list failed" { + t.Fatalf("expected engine error unchanged, got %v", err) + } +} + +func installCurrentFileCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { + t.Helper() + originalConnect := connectListSearchDBPhase + originalNewEngine := newCommandEngine + t.Cleanup(func() { + connectListSearchDBPhase = originalConnect + newCommandEngine = originalNewEngine + }) + connectListSearchDBPhase = func() (*sql.DB, error) { return dbconn, nil } + newCommandEngine = func(*sql.DB, string) (engine.Engine, error) { return stub, nil } +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 4416e2db..f1221f2c 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -40,7 +40,6 @@ import ( internalgc "github.com/franchoy/coldkeep/internal/gc" "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/iodebug" - "github.com/franchoy/coldkeep/internal/listing" "github.com/franchoy/coldkeep/internal/maintenance" "github.com/franchoy/coldkeep/internal/observability" "github.com/franchoy/coldkeep/internal/recovery" @@ -2841,35 +2840,36 @@ func runListCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { if len(parsed.positionals) != 0 { return usageErrorf("Usage: coldkeep list [--limit ] [--offset ]") } - dbconn, err := db.ConnectDB() + req, err := listFilesRequest(parsed) + if err != nil { + return err + } + dbconn, err := connectListSearchDBPhase() if err != nil { return fmt.Errorf("failed to connect to DB: %w", err) } defer func() { _ = dbconn.Close() }() + eng, err := newCommandEngine(dbconn, "") + if err != nil { + return err + } + result, err := eng.ListFiles(context.Background(), req) + if err != nil { + return err + } if outputMode == outputModeJSON { - files, err := listing.ListFilesResultWithDB(dbconn, listArgs(parsed)) - if err != nil { - return err - } - if files == nil { - files = []listing.FileRecord{} - } payload := map[string]any{ "status": "ok", "command": "list", - "files": files, + "files": result.Files, } encoded, _ := json.Marshal(payload) fmt.Println(string(encoded)) return nil } - files, err := listing.ListFilesResultWithDB(dbconn, listArgs(parsed)) - if err != nil { - return err - } - printFileRecordsTable(files) + printFileRecordsTable(result.Files) return nil } @@ -2897,39 +2897,40 @@ func runSearchCommand(parsed parsedCommandLine, outputMode cliOutputMode) error if err := validateNonNegativeIntegerFlag(parsed, "offset"); err != nil { return err } - dbconn, err := db.ConnectDB() + req, err := searchFilesRequest(parsed) + if err != nil { + return err + } + dbconn, err := connectListSearchDBPhase() if err != nil { return fmt.Errorf("failed to connect to DB: %w", err) } defer func() { _ = dbconn.Close() }() + eng, err := newCommandEngine(dbconn, "") + if err != nil { + return err + } + result, err := eng.SearchFiles(context.Background(), req) + if err != nil { + return err + } if outputMode == outputModeJSON { - files, err := listing.SearchFilesResultWithDB(dbconn, searchArgs(parsed)) - if err != nil { - return err - } - if files == nil { - files = []listing.FileRecord{} - } payload := map[string]any{ "status": "ok", "command": "search", - "files": files, + "files": result.Files, } encoded, _ := json.Marshal(payload) fmt.Println(string(encoded)) return nil } - files, err := listing.SearchFilesResultWithDB(dbconn, searchArgs(parsed)) - if err != nil { - return err - } - printFileRecordsTable(files) + printFileRecordsTable(result.Files) return nil } -func printFileRecordsTable(records []listing.FileRecord) { +func printFileRecordsTable(records []engine.CurrentFile) { fmt.Printf("%-6s %-25s %-15s %-20s\n", "ID", "PATH", "SIZE(bytes)", "CREATED_AT") fmt.Println("---------------------------------------------------------------------") for _, r := range records { @@ -7322,48 +7323,74 @@ func validateNonNegativeIntegerFlag(parsed parsedCommandLine, name string) error if err != nil || parsedValue < 0 { return usageErrorf("invalid --%s value %q: must be a non-negative integer", name, value) } - if name == "limit" && parsedValue > listing.MaxPaginationLimit { - return usageErrorf("invalid --%s value %q: must be <= %d", name, value, listing.MaxPaginationLimit) + if name == "limit" && parsedValue > engine.MaxFileQueryLimit { + return usageErrorf("invalid --%s value %q: must be <= %d", name, value, engine.MaxFileQueryLimit) } return nil } -func listArgs(parsed parsedCommandLine) []string { - args := make([]string, 0, 4) - - if value, ok := parsed.lastFlagValue("limit"); ok { - args = append(args, "--limit", value) +func listFilesRequest(parsed parsedCommandLine) (engine.ListFilesRequest, error) { + limit, err := parsedInt64Pointer(parsed, "limit") + if err != nil { + return engine.ListFilesRequest{}, err } - if value, ok := parsed.lastFlagValue("offset"); ok { - args = append(args, "--offset", value) + offset, err := parsedInt64Pointer(parsed, "offset") + if err != nil { + return engine.ListFilesRequest{}, err } - - return args + return engine.ListFilesRequest{Limit: limit, Offset: offset}, nil } -func searchArgs(parsed parsedCommandLine) []string { - orderedFlags := []string{"name", "min-size", "max-size"} - args := make([]string, 0) - - for _, flag := range orderedFlags { - for _, value := range parsed.flags[flag] { - args = append(args, "--"+flag) - if value != "" { - args = append(args, value) - } - } +func searchFilesRequest(parsed parsedCommandLine) (engine.SearchFilesRequest, error) { + limit, err := parsedInt64Pointer(parsed, "limit") + if err != nil { + return engine.SearchFilesRequest{}, err + } + offset, err := parsedInt64Pointer(parsed, "offset") + if err != nil { + return engine.SearchFilesRequest{}, err + } + mins, err := parsedInt64Values(parsed, "min-size") + if err != nil { + return engine.SearchFilesRequest{}, err } + maxes, err := parsedInt64Values(parsed, "max-size") + if err != nil { + return engine.SearchFilesRequest{}, err + } + return engine.SearchFilesRequest{ + NameContains: append([]string(nil), parsed.flags["name"]...), + MinSizeBytes: mins, + MaxSizeBytes: maxes, + Limit: limit, + Offset: offset, + }, nil +} - if value, ok := parsed.lastFlagValue("limit"); ok { - args = append(args, "--limit", value) +func parsedInt64Pointer(parsed parsedCommandLine, name string) (*int64, error) { + value, ok := parsed.lastFlagValue(name) + if !ok { + return nil, nil } - if value, ok := parsed.lastFlagValue("offset"); ok { - args = append(args, "--offset", value) + parsedValue, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, usageErrorf("invalid --%s value %q: must be a non-negative integer", name, value) } + return &parsedValue, nil +} - args = append(args, parsed.positionals...) - return args +func parsedInt64Values(parsed parsedCommandLine, name string) ([]int64, error) { + values := parsed.flags[name] + result := make([]int64, 0, len(values)) + for _, value := range values { + parsedValue, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, usageErrorf("invalid --%s value %q: must be a non-negative integer", name, value) + } + result = append(result, parsedValue) + } + return result, nil } func parseVerifyLevel(parsed parsedCommandLine) (verify.VerifyLevel, error) { diff --git a/cmd/coldkeep/main_test.go b/cmd/coldkeep/main_test.go index 5102ca5a..53c3863a 100644 --- a/cmd/coldkeep/main_test.go +++ b/cmd/coldkeep/main_test.go @@ -3487,8 +3487,8 @@ func runParsedCommandForParserPathValidation(parsed parsedCommandLine) error { } } -func TestSearchArgsIncludesPaginationFlags(t *testing.T) { - args := searchArgs(parsedCommandLine{ +func TestSearchFilesRequestUsesLastPaginationValues(t *testing.T) { + req, err := searchFilesRequest(parsedCommandLine{ method: "search", flags: map[string][]string{ "name": {"report"}, @@ -3497,28 +3497,27 @@ func TestSearchArgsIncludesPaginationFlags(t *testing.T) { }, positionals: []string{"ignored-positional"}, }) - - encoded := strings.Join(args, " ") - if !strings.Contains(encoded, "--limit 50") { - t.Fatalf("expected last limit value to be forwarded, got %q", encoded) + if err != nil { + t.Fatalf("searchFilesRequest: %v", err) } - if strings.Contains(encoded, "--limit 25") { - t.Fatalf("expected earlier limit values to be ignored, got %q", encoded) + if req.Limit == nil || *req.Limit != 50 { + t.Fatalf("expected last limit value to be forwarded, got %+v", req.Limit) } - if !strings.Contains(encoded, "--offset 100") { - t.Fatalf("expected offset value to be forwarded, got %q", encoded) + if req.Offset == nil || *req.Offset != 100 { + t.Fatalf("expected offset value to be forwarded, got %+v", req.Offset) } } -func TestSearchArgsPreservesValidNameValue(t *testing.T) { - args := searchArgs(parsedCommandLine{ +func TestSearchFilesRequestPreservesValidNameValue(t *testing.T) { + req, err := searchFilesRequest(parsedCommandLine{ method: "search", flags: map[string][]string{"name": {"report"}}, }) - - encoded := strings.Join(args, " ") - if encoded != "--name report" { - t.Fatalf("expected valid search name to be forwarded unchanged, got %q", encoded) + if err != nil { + t.Fatalf("searchFilesRequest: %v", err) + } + if len(req.NameContains) != 1 || req.NameContains[0] != "report" { + t.Fatalf("expected valid search name to be forwarded unchanged, got %+v", req.NameContains) } } diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index 0eb92537..16261242 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -8,11 +8,14 @@ import ( "strings" "github.com/franchoy/coldkeep/internal/batch" + "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/storage" ) +var connectListSearchDBPhase = db.ConnectDB + var newCommandEngine = func(dbconn *sql.DB, containerDir string) (engine.Engine, error) { return engine.New(engine.Config{DB: dbconn, ContainerDir: containerDir}) } diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 3339a23b..91b7c2e5 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -23,6 +23,8 @@ type stubCommandEngine struct { restoreStoredPathFunc func(context.Context, engine.RestoreStoredPathRequest) (engine.RestoreStoredPathResult, error) removeStoredPathsFunc func(context.Context, engine.RemoveStoredPathsRequest) (engine.RemoveStoredPathsResult, error) storeFolderFunc func(context.Context, engine.StoreFolderRequest) (engine.StoreFolderResult, error) + listFilesFunc func(context.Context, engine.ListFilesRequest) (engine.ListFilesResult, error) + searchFilesFunc func(context.Context, engine.SearchFilesRequest) (engine.SearchFilesResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -67,6 +69,20 @@ func (s stubCommandEngine) StoreFolder(ctx context.Context, req engine.StoreFold return engine.StoreFolderResult{}, errors.New("unexpected StoreFolder call") } +func (s stubCommandEngine) ListFiles(ctx context.Context, req engine.ListFilesRequest) (engine.ListFilesResult, error) { + if s.listFilesFunc != nil { + return s.listFilesFunc(ctx, req) + } + return engine.ListFilesResult{}, errors.New("unexpected ListFiles call") +} + +func (s stubCommandEngine) SearchFiles(ctx context.Context, req engine.SearchFilesRequest) (engine.SearchFilesResult, error) { + if s.searchFilesFunc != nil { + return s.searchFilesFunc(ctx, req) + } + return engine.SearchFilesResult{}, errors.New("unexpected SearchFiles call") +} + func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 0f369b60..6ad4bb80 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -75,11 +75,13 @@ ## Phase 11 — List and Search Ownership -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 11 list/search ownership](v1.13.12-phase11-list-and-search-ownership.md). ## Phase 12 — Configuration Ownership -**Status:** Not started +**Status:** Next ## Phase 13 — Snapshot Read-Side Completion diff --git a/docs/release/v1.13/v1.13.12-phase11-list-and-search-ownership.md b/docs/release/v1.13/v1.13.12-phase11-list-and-search-ownership.md new file mode 100644 index 00000000..e3ecc29e --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase11-list-and-search-ownership.md @@ -0,0 +1,64 @@ +# Coldkeep v1.13.12 Phase 11 — List and Search Ownership + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `11 — List and Search Ownership` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Catalog and engine contracts + +`CurrentFileCatalog` now exposes distinct typed list and search operations for +completed current-state physical paths. Neutral filters preserve repeated name, +minimum-size, and maximum-size constraints plus optional limit and offset. +Results retain logical ID, stored path, hash, logical byte size, and creation +time without exposing SQL, database handles, or CLI tokens. + +`Engine.ListFiles` and `Engine.SearchFiles` validate pagination and filter +invariants, translate catalog failures into typed engine errors, and project the +established timestamp and JSON field representation. Direct engine tests cover +both methods, filters, validation, empty behavior, ordering, and cancellation. + +## Production routing + +The production `list` and `search` commands now parse their public flags into +typed requests, construct the injected engine, invoke exactly one engine +operation, and render its neutral result. The raw argument-array-aware listing +package and its duplicate SQL implementation were removed. The integration +consistency fixture now exercises the engine contract directly. + +The CLI retains usage validation and the exact table/JSON presentation. Public +syntax, case-insensitive substring/wildcard matching, repeated-filter AND +semantics, completed-file selection, timestamps, empty JSON arrays, and error +projection are unchanged. + +## Backend-neutral ordering + +The shared contract exposed a collation difference: PostgreSQL and SQLite did +not sort punctuation in stored paths identically. Catalog queries now apply +filters first, then perform byte-stable path ordering with logical-ID tie +breaking and pagination in the neutral catalog layer. This makes ordering and +page boundaries deterministic across supported backends. + +## Validation + +- Direct catalog and engine list/search suites: pass. +- CLI typed-request, text/JSON parity, and error-propagation suites: pass. +- Shared SQLite/PostgreSQL current-file contract: pass. +- Focused catalog, engine, and CLI race suites: pass. +- Full `go test -count=1 ./...`: pass. +- Validation root: `/tmp/coldkeep-v11312-phase11.0MWTEm/`, containing isolated + caches, temporary files, storage roots, and SQLite databases. + +PostgreSQL validation created and dropped exactly +`coldkeep_bt_testcatalogcontractcurrentfilequeriesacros_2a9281_1` during the +initial fail-closed collation check and +`coldkeep_bt_testcatalogcontractcurrentfilequeriesacros_2a95f0_1` during the +passing corrected run. A read-only maintenance query confirmed both exact names +are absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 12 is authorized to activate repository configuration catalog and engine +ownership without changing schema, backend defaults, or public configuration +semantics. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 14bac469..256637c4 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–10 are Complete. Phase 11 is Next: add typed catalog and engine -current-file list/search operations while preserving filtering, normalization, -sorting, pagination, JSON, text, and stored-path semantics. +Phases 0–11 are Complete. Phase 12 is Next: activate typed catalog and engine +repository-configuration ownership while preserving keys, defaults, validation, +changed/unchanged semantics, metadata, and public output. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 17156bd4..56011ede 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -170,11 +170,23 @@ ## Phase 11 — List and Search Ownership -**Phase status:** Next +**Phase status:** Complete + +- [x] Catalog owns typed completed-current-file list and search queries. +- [x] Repeated name/minimum/maximum filters, bounded pagination, completed-file + selection, and exact metadata projection are directly tested. +- [x] Final path ordering and pagination are deterministic and identical on + SQLite and PostgreSQL regardless of database collation. +- [x] `Engine.ListFiles` and `Engine.SearchFiles` validate typed requests and + return neutral current-file DTOs with frozen JSON field names. +- [x] Production CLI list/search routes only through engine operations; the raw + CLI-token-aware listing package and duplicate SQL are removed. +- [x] Text/JSON compatibility, engine errors, cancellation, read-only behavior, + race coverage, and dual-backend catalog parity pass. ## Phase 12 — Configuration Ownership -**Phase status:** Not started +**Phase status:** Next ## Phase 13 — Snapshot Read-Side Completion diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index bae364ea..63bb0709 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -170,6 +170,62 @@ func TestCatalogContractFindPhysicalFilesAcrossBackends(t *testing.T) { }) } +func TestCatalogContractCurrentFileQueriesAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + seedCatalogFixture(t, backend.DB) + svc := catalog.NewServiceFromSQL(backend.DB) + before := catalogStateCounts(t, backend.DB) + + all, err := svc.ListCurrentFiles(context.Background(), catalog.CurrentFilePage{}) + if err != nil { + t.Fatalf("ListCurrentFiles: %v", err) + } + paths := make([]string, len(all)) + for i, ref := range all { + paths[i] = ref.Path + if ref.CreatedAt.IsZero() { + t.Fatalf("current file has zero creation time: %+v", ref) + } + } + if want := []string{"/current/a.txt", "/current/b.txt", "/current/both.txt"}; !reflect.DeepEqual(paths, want) { + t.Fatalf("current-file ordering/filtering: got %v want %v", paths, want) + } + + limit, offset := int64(1), int64(1) + page, err := svc.ListCurrentFiles(context.Background(), catalog.CurrentFilePage{Limit: &limit, Offset: &offset}) + if err != nil || len(page) != 1 || page[0].Path != "/current/b.txt" { + t.Fatalf("paginated current files: got (%+v, %v)", page, err) + } + offsetOnly, err := svc.ListCurrentFiles(context.Background(), catalog.CurrentFilePage{Offset: &offset}) + if err != nil || len(offsetOnly) != 2 || offsetOnly[0].Path != "/current/b.txt" { + t.Fatalf("offset-only current files: got (%+v, %v)", offsetOnly, err) + } + + matches, err := svc.SearchCurrentFiles(context.Background(), catalog.CurrentFileSearch{ + NameContains: []string{"current", "both"}, + MinSizeBytes: []int64{30}, + MaxSizeBytes: []int64{40}, + }) + if err != nil || len(matches) != 1 || matches[0].Path != "/current/both.txt" || matches[0].LogicalFileID != 3 { + t.Fatalf("searched current files: got (%+v, %v)", matches, err) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if refs, err := svc.ListCurrentFiles(cancelled, catalog.CurrentFilePage{}); refs != nil || !catalog.IsCode(err, catalog.ErrorCancelled) { + t.Fatalf("cancelled list current files: got (%+v, %v)", refs, err) + } + negative := int64(-1) + if refs, err := svc.SearchCurrentFiles(context.Background(), catalog.CurrentFileSearch{Page: catalog.CurrentFilePage{Limit: &negative}}); refs != nil || !catalog.IsCode(err, catalog.ErrorInvalidArgument) { + t.Fatalf("invalid current-file page: got (%+v, %v)", refs, err) + } + + if after := catalogStateCounts(t, backend.DB); after != before { + t.Fatalf("current-file reads mutated catalog state: before=%+v after=%+v", before, after) + } + }) +} + func requirePhysicalFiles(t *testing.T, svc interface { FindPhysicalFilesForLogicalFile(context.Context, int64) ([]catalog.PhysicalFileRef, error) }, id int64) []catalog.PhysicalFileRef { diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 367f2a8a..9199504a 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -68,6 +68,12 @@ type PhysicalFileCatalog interface { FindPhysicalFilesForLogicalFile(ctx context.Context, logicalFileID int64) ([]PhysicalFileRef, error) } +// CurrentFileCatalog owns completed current-state path query truth. +type CurrentFileCatalog interface { + ListCurrentFiles(ctx context.Context, page CurrentFilePage) ([]CurrentFileRef, error) + SearchCurrentFiles(ctx context.Context, filter CurrentFileSearch) ([]CurrentFileRef, error) +} + // SnapshotCatalog exposes snapshot metadata. type SnapshotCatalog interface { // FindSnapshot returns the snapshot with the given ID. @@ -134,6 +140,7 @@ type GCPlanCatalog interface { type Catalog interface { LogicalFileCatalog PhysicalFileCatalog + CurrentFileCatalog SnapshotCatalog SnapshotGraphCatalog ReachabilityCatalog diff --git a/internal/catalog/current_files.go b/internal/catalog/current_files.go new file mode 100644 index 00000000..6a373599 --- /dev/null +++ b/internal/catalog/current_files.go @@ -0,0 +1,137 @@ +package catalog + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + filestate "github.com/franchoy/coldkeep/internal/status" +) + +func (s *Service) ListCurrentFiles(ctx context.Context, page CurrentFilePage) ([]CurrentFileRef, error) { + if err := validateCurrentFilePage(page); err != nil { + return nil, err + } + return s.queryCurrentFiles(ctx, CurrentFileSearch{Page: page}, "list current files") +} + +func (s *Service) SearchCurrentFiles(ctx context.Context, filter CurrentFileSearch) ([]CurrentFileRef, error) { + if err := validateCurrentFileSearch(filter); err != nil { + return nil, err + } + return s.queryCurrentFiles(ctx, filter, "search current files") +} + +func validateCurrentFileSearch(filter CurrentFileSearch) error { + if err := validateCurrentFilePage(filter.Page); err != nil { + return err + } + for _, name := range filter.NameContains { + if strings.TrimSpace(name) == "" { + return NewError(ErrorInvalidArgument, "search current files", "name_filter_nonblank", "name filter cannot be empty", nil) + } + } + for _, size := range filter.MinSizeBytes { + if size < 0 { + return NewError(ErrorInvalidArgument, "search current files", "minimum_size_nonnegative", "minimum size must be non-negative", nil) + } + } + for _, size := range filter.MaxSizeBytes { + if size < 0 { + return NewError(ErrorInvalidArgument, "search current files", "maximum_size_nonnegative", "maximum size must be non-negative", nil) + } + } + return nil +} + +func validateCurrentFilePage(page CurrentFilePage) error { + if page.Limit != nil { + if *page.Limit < 0 { + return NewError(ErrorInvalidArgument, "query current files", "limit_nonnegative", "limit must be non-negative", nil) + } + if *page.Limit > MaxCurrentFilePageSize { + return NewError(ErrorInvalidArgument, "query current files", "limit_bounded", fmt.Sprintf("limit must be <= %d", MaxCurrentFilePageSize), nil) + } + } + if page.Offset != nil && *page.Offset < 0 { + return NewError(ErrorInvalidArgument, "query current files", "offset_nonnegative", "offset must be non-negative", nil) + } + return nil +} + +func (s *Service) queryCurrentFiles(ctx context.Context, filter CurrentFileSearch, operation string) ([]CurrentFileRef, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, currentFileCatalogError(operation, err) + } + + var query strings.Builder + query.WriteString(` +SELECT lf.id, pf.path, lf.file_hash, lf.total_size, lf.created_at +FROM physical_file pf +JOIN logical_file lf ON lf.id = pf.logical_file_id +WHERE lf.status = $1`) + params := []any{filestate.LogicalFileCompleted} + for _, name := range filter.NameContains { + query.WriteString(fmt.Sprintf(" AND LOWER(pf.path) LIKE LOWER($%d)", len(params)+1)) + params = append(params, "%"+name+"%") + } + for _, size := range filter.MinSizeBytes { + query.WriteString(fmt.Sprintf(" AND lf.total_size >= $%d", len(params)+1)) + params = append(params, size) + } + for _, size := range filter.MaxSizeBytes { + query.WriteString(fmt.Sprintf(" AND lf.total_size <= $%d", len(params)+1)) + params = append(params, size) + } + + rows, err := s.db.QueryContext(ctx, query.String(), params...) + if err != nil { + return nil, currentFileCatalogError(operation, err) + } + defer func() { _ = rows.Close() }() + + refs := make([]CurrentFileRef, 0) + for rows.Next() { + var ref CurrentFileRef + if err := rows.Scan(&ref.LogicalFileID, &ref.Path, &ref.FileHash, &ref.SizeBytes, &ref.CreatedAt); err != nil { + return nil, currentFileCatalogError(operation, err) + } + refs = append(refs, ref) + } + if err := rows.Err(); err != nil { + return nil, currentFileCatalogError(operation, err) + } + sort.Slice(refs, func(i, j int) bool { + if refs[i].Path != refs[j].Path { + return refs[i].Path < refs[j].Path + } + return refs[i].LogicalFileID < refs[j].LogicalFileID + }) + return paginateCurrentFiles(refs, filter.Page), nil +} + +func paginateCurrentFiles(refs []CurrentFileRef, page CurrentFilePage) []CurrentFileRef { + start := 0 + if page.Offset != nil && *page.Offset < int64(len(refs)) { + start = int(*page.Offset) + } else if page.Offset != nil { + start = len(refs) + } + end := len(refs) + if page.Limit != nil && *page.Limit < int64(end-start) { + end = start + int(*page.Limit) + } + return refs[start:end] +} + +func currentFileCatalogError(operation string, err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return NewError(ErrorCancelled, operation, "", operation+" cancelled", err) + } + return NewError(ErrorOperationFailed, operation, "", operation+" query failed", err) +} diff --git a/internal/catalog/service.go b/internal/catalog/service.go index ff1ff0a5..bb41e45c 100644 --- a/internal/catalog/service.go +++ b/internal/catalog/service.go @@ -16,6 +16,7 @@ var ( _ Catalog = (*Service)(nil) _ LogicalFileCatalog = (*Service)(nil) _ PhysicalFileCatalog = (*Service)(nil) + _ CurrentFileCatalog = (*Service)(nil) _ SnapshotCatalog = (*Service)(nil) _ SnapshotGraphCatalog = (*Service)(nil) _ ReachabilityCatalog = (*Service)(nil) diff --git a/internal/catalog/types.go b/internal/catalog/types.go index c6b3420a..093d8f5e 100644 --- a/internal/catalog/types.go +++ b/internal/catalog/types.go @@ -21,6 +21,32 @@ type PhysicalFileRef struct { IsMetadataComplete bool } +const MaxCurrentFilePageSize int64 = 10000 + +// CurrentFileRef is one completed current-state physical path joined to its +// logical identity. Snapshot-only and non-completed logical files are excluded. +type CurrentFileRef struct { + LogicalFileID int64 + Path string + FileHash string + SizeBytes int64 + CreatedAt time.Time +} + +type CurrentFilePage struct { + Limit *int64 + Offset *int64 +} + +// CurrentFileSearch preserves repeated CLI filters. Repeated name and size +// constraints are combined with AND, matching the historical query behavior. +type CurrentFileSearch struct { + NameContains []string + MinSizeBytes []int64 + MaxSizeBytes []int64 + Page CurrentFilePage +} + type SnapshotRef struct { ID string Type string diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index a05ab380..fb88b20e 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -157,6 +157,41 @@ type StoreFolderResult struct { WorkersUsed int } +const MaxFileQueryLimit int64 = 10000 + +// CurrentFile is a presentation-neutral completed current-state path. +// JSON tags preserve the established CLI projection when the CLI embeds it. +type CurrentFile struct { + ID int64 `json:"id"` + Name string `json:"name"` + FileHash string `json:"file_hash"` + SizeBytes int64 `json:"size_bytes"` + CreatedAt string `json:"created_at"` +} + +type ListFilesRequest struct { + Limit *int64 + Offset *int64 +} + +type ListFilesResult struct { + Files []CurrentFile +} + +// SearchFilesRequest preserves repeated filters and their historical AND +// semantics without exposing raw CLI tokens to the engine or catalog. +type SearchFilesRequest struct { + NameContains []string + MinSizeBytes []int64 + MaxSizeBytes []int64 + Limit *int64 + Offset *int64 +} + +type SearchFilesResult struct { + Files []CurrentFile +} + // --------------------------------------------------------------------------- // Restore // --------------------------------------------------------------------------- diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index f5a18ff0..3ae9c9ea 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index b3b8af77..16cd2d69 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -44,6 +44,11 @@ func activeCoreContractTypes() []struct { {"StoreResult", engine.StoreResult{}}, {"StoreFolderRequest", engine.StoreFolderRequest{}}, {"StoreFolderResult", engine.StoreFolderResult{}}, + {"CurrentFile", engine.CurrentFile{}}, + {"ListFilesRequest", engine.ListFilesRequest{}}, + {"ListFilesResult", engine.ListFilesResult{}}, + {"SearchFilesRequest", engine.SearchFilesRequest{}}, + {"SearchFilesResult", engine.SearchFilesResult{}}, {"RestoreRequest", engine.RestoreRequest{}}, {"RestoreItemResult", engine.RestoreItemResult{}}, {"RestoreResult", engine.RestoreResult{}}, diff --git a/internal/engine/current_files.go b/internal/engine/current_files.go new file mode 100644 index 00000000..59ae8aad --- /dev/null +++ b/internal/engine/current_files.go @@ -0,0 +1,88 @@ +package engine + +import ( + "context" + "fmt" + "strings" + + "github.com/franchoy/coldkeep/internal/catalog" +) + +func (e *DefaultEngine) ListFiles(ctx context.Context, req ListFilesRequest) (ListFilesResult, error) { + if err := validateFileQueryPage(req.Limit, req.Offset); err != nil { + return ListFilesResult{}, TranslateErrorAs("list files", ErrorInvalidArgument, err) + } + refs, err := catalog.NewServiceFromSQL(e.config.DB).ListCurrentFiles(ctx, catalog.CurrentFilePage{ + Limit: req.Limit, Offset: req.Offset, + }) + if err != nil { + return ListFilesResult{}, TranslateError("list files", err) + } + return ListFilesResult{Files: currentFilesFromCatalog(refs)}, nil +} + +func (e *DefaultEngine) SearchFiles(ctx context.Context, req SearchFilesRequest) (SearchFilesResult, error) { + if err := validateSearchFilesRequest(req); err != nil { + return SearchFilesResult{}, TranslateErrorAs("search files", ErrorInvalidArgument, err) + } + refs, err := catalog.NewServiceFromSQL(e.config.DB).SearchCurrentFiles(ctx, catalog.CurrentFileSearch{ + NameContains: append([]string(nil), req.NameContains...), + MinSizeBytes: append([]int64(nil), req.MinSizeBytes...), + MaxSizeBytes: append([]int64(nil), req.MaxSizeBytes...), + Page: catalog.CurrentFilePage{ + Limit: req.Limit, Offset: req.Offset, + }, + }) + if err != nil { + return SearchFilesResult{}, TranslateError("search files", err) + } + return SearchFilesResult{Files: currentFilesFromCatalog(refs)}, nil +} + +func validateSearchFilesRequest(req SearchFilesRequest) error { + if err := validateFileQueryPage(req.Limit, req.Offset); err != nil { + return err + } + for _, name := range req.NameContains { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("engine: search files name filter cannot be empty") + } + } + for _, size := range req.MinSizeBytes { + if size < 0 { + return fmt.Errorf("engine: search files minimum size must be non-negative") + } + } + for _, size := range req.MaxSizeBytes { + if size < 0 { + return fmt.Errorf("engine: search files maximum size must be non-negative") + } + } + return nil +} + +func validateFileQueryPage(limit, offset *int64) error { + if limit != nil { + if *limit < 0 { + return fmt.Errorf("engine: file query limit must be non-negative") + } + if *limit > MaxFileQueryLimit { + return fmt.Errorf("engine: file query limit must be <= %d", MaxFileQueryLimit) + } + } + if offset != nil && *offset < 0 { + return fmt.Errorf("engine: file query offset must be non-negative") + } + return nil +} + +func currentFilesFromCatalog(refs []catalog.CurrentFileRef) []CurrentFile { + files := make([]CurrentFile, len(refs)) + for i, ref := range refs { + files[i] = CurrentFile{ + ID: ref.LogicalFileID, Name: ref.Path, FileHash: ref.FileHash, + SizeBytes: ref.SizeBytes, CreatedAt: ref.CreatedAt.Format("2006-01-02 15:04:05"), + } + } + return files +} diff --git a/internal/engine/current_files_test.go b/internal/engine/current_files_test.go new file mode 100644 index 00000000..2f0562c3 --- /dev/null +++ b/internal/engine/current_files_test.go @@ -0,0 +1,83 @@ +package engine_test + +import ( + "context" + "database/sql" + "errors" + "reflect" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + filestate "github.com/franchoy/coldkeep/internal/status" +) + +func TestEngineListAndSearchFiles(t *testing.T) { + dbconn := openSnapshotTestDB(t) + seedEngineCurrentFile(t, dbconn, 1, "/docs/report.txt", "hash-report", 120, filestate.LogicalFileCompleted) + seedEngineCurrentFile(t, dbconn, 2, "/docs/notes.txt", "hash-notes", 40, filestate.LogicalFileCompleted) + seedEngineCurrentFile(t, dbconn, 3, "/docs/aborted.txt", "hash-aborted", 80, filestate.LogicalFileAborted) + eng, err := engine.New(engine.Config{DB: dbconn}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + + listed, err := eng.ListFiles(context.Background(), engine.ListFilesRequest{}) + if err != nil { + t.Fatalf("ListFiles: %v", err) + } + if got := currentFileNames(listed.Files); !reflect.DeepEqual(got, []string{"/docs/notes.txt", "/docs/report.txt"}) { + t.Fatalf("ListFiles paths: %v", got) + } + if listed.Files[1].ID != 1 || listed.Files[1].FileHash != "hash-report" || listed.Files[1].SizeBytes != 120 || listed.Files[1].CreatedAt == "" { + t.Fatalf("ListFiles projection: %+v", listed.Files[1]) + } + + searched, err := eng.SearchFiles(context.Background(), engine.SearchFilesRequest{ + NameContains: []string{"docs", "report"}, MinSizeBytes: []int64{100}, MaxSizeBytes: []int64{200}, + }) + if err != nil || len(searched.Files) != 1 || searched.Files[0].Name != "/docs/report.txt" { + t.Fatalf("SearchFiles: got (%+v, %v)", searched, err) + } +} + +func TestEngineCurrentFileValidationAndCancellation(t *testing.T) { + dbconn := openSnapshotTestDB(t) + eng, err := engine.New(engine.Config{DB: dbconn}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + negative := int64(-1) + if _, err := eng.ListFiles(context.Background(), engine.ListFilesRequest{Limit: &negative}); !engine.IsCode(err, engine.ErrorInvalidArgument) { + t.Fatalf("negative ListFiles limit: %v", err) + } + if _, err := eng.SearchFiles(context.Background(), engine.SearchFilesRequest{NameContains: []string{" "}}); !engine.IsCode(err, engine.ErrorInvalidArgument) { + t.Fatalf("blank SearchFiles name: %v", err) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := eng.ListFiles(cancelled, engine.ListFilesRequest{}); !errors.Is(err, context.Canceled) || !engine.IsCode(err, engine.ErrorCancelled) { + t.Fatalf("cancelled ListFiles: %v", err) + } +} + +func seedEngineCurrentFile(t *testing.T, dbconn *sql.DB, id int64, path, hash string, size int64, status string) { + t.Helper() + if _, err := dbconn.Exec(` +INSERT INTO logical_file (id, original_name, total_size, file_hash, status, ref_count, chunker_version) +VALUES ($1, $2, $3, $4, $5, 1, 'v1-simple-rolling')`, id, path, size, hash, status); err != nil { + t.Fatalf("insert logical file: %v", err) + } + if _, err := dbconn.Exec(` +INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) +VALUES ($1, $2, 1)`, path, id); err != nil { + t.Fatalf("insert physical file: %v", err) + } +} + +func currentFileNames(files []engine.CurrentFile) []string { + names := make([]string, len(files)) + for i, file := range files { + names[i] = file.Name + } + return names +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 7dd51aef..cedfb5f3 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -84,6 +84,12 @@ type Engine interface { // below this engine boundary. StoreFolder(ctx context.Context, req StoreFolderRequest) (StoreFolderResult, error) + // ListFiles returns completed current-state stored paths in path order. + ListFiles(ctx context.Context, req ListFilesRequest) (ListFilesResult, error) + + // SearchFiles returns completed current-state paths matching typed filters. + SearchFiles(ctx context.Context, req SearchFilesRequest) (SearchFilesResult, error) + // Remove removes logical files from the repository by logical file ID. // Safety invariant: Remove must never make valid data unrecoverable. // Method selection owns addressing semantics: this method is by-ID only. diff --git a/internal/listing/list.go b/internal/listing/list.go deleted file mode 100644 index fda88165..00000000 --- a/internal/listing/list.go +++ /dev/null @@ -1,74 +0,0 @@ -package listing - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/franchoy/coldkeep/internal/db" - filestate "github.com/franchoy/coldkeep/internal/status" -) - -// FileRecord is a single current-state physical file entry returned by list/search. -type FileRecord struct { - ID int64 `json:"id"` - Name string `json:"name"` - FileHash string `json:"file_hash"` - SizeBytes int64 `json:"size_bytes"` - CreatedAt string `json:"created_at"` -} - -// ListFilesResultWithDB returns raw records using a caller-managed DB connection. -func ListFilesResultWithDB(dbconn *sql.DB, args []string) ([]FileRecord, error) { - if dbconn == nil { - return nil, fmt.Errorf("db connection is nil") - } - - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() - - limit, offset, err := parsePaginationArgs(args) - if err != nil { - return nil, err - } - - query := ` - SELECT lf.id, pf.path, lf.file_hash, lf.total_size, lf.created_at - FROM physical_file pf - JOIN logical_file lf ON lf.id = pf.logical_file_id - WHERE lf.status = $1 - ORDER BY pf.path ASC - ` - params := []interface{}{filestate.LogicalFileCompleted} - query, params = applyPagination(query, params, 2, limit, offset) - - rows, err := dbconn.QueryContext(ctx, query, params...) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var records []FileRecord - for rows.Next() { - var id int64 - var name string - var fileHash string - var size int64 - var created time.Time - if err := rows.Scan(&id, &name, &fileHash, &size, &created); err != nil { - return nil, err - } - records = append(records, FileRecord{ - ID: id, - Name: name, - FileHash: fileHash, - SizeBytes: size, - CreatedAt: created.Format("2006-01-02 15:04:05"), - }) - } - if err := rows.Err(); err != nil { - return nil, err - } - return records, nil -} diff --git a/internal/listing/list_test.go b/internal/listing/list_test.go deleted file mode 100644 index 6995e496..00000000 --- a/internal/listing/list_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package listing - -import ( - "database/sql" - "strings" - "testing" - - "github.com/franchoy/coldkeep/internal/db" - filestate "github.com/franchoy/coldkeep/internal/status" - - _ "github.com/mattn/go-sqlite3" -) - -func openListTestDB(t *testing.T) *sql.DB { - t.Helper() - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("open test db: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func TestListFilesFailsWhenDBIsNil(t *testing.T) { - _, err := ListFilesResultWithDB(nil, nil) - if err == nil || !strings.Contains(err.Error(), "db connection is nil") { - t.Fatalf("expected db-nil error contract, got: %v", err) - } -} - -func TestListFilesFailsWhenLimitArgIsMissing(t *testing.T) { - _, err := ListFilesResultWithDB(openListTestDB(t), []string{"--limit"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --limit") { - t.Fatalf("expected missing-limit error contract, got: %v", err) - } -} - -func TestListFilesFailsWhenOffsetArgIsInvalid(t *testing.T) { - _, err := ListFilesResultWithDB(openListTestDB(t), []string{"--offset", "-1"}) - if err == nil || !strings.Contains(err.Error(), "invalid --offset") { - t.Fatalf("expected invalid-offset error contract, got: %v", err) - } -} - -func TestListFilesFailsWhenLimitArgExceedsMaximum(t *testing.T) { - _, err := ListFilesResultWithDB(openListTestDB(t), []string{"--limit", "10001"}) - if err == nil || !strings.Contains(err.Error(), "invalid --limit") { - t.Fatalf("expected invalid-limit error contract, got: %v", err) - } -} - -func TestListFilesReturnsCurrentStatePhysicalPaths(t *testing.T) { - dbconn := openListTestDB(t) - if err := db.RunMigrations(dbconn); err != nil { - t.Fatalf("run migrations: %v", err) - } - - insertLogical := func(name, hash string, status string) int64 { - t.Helper() - var id int64 - if err := dbconn.QueryRow( - `INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) - VALUES ($1, $2, $3, $4, $5, 'v1-simple-rolling') - RETURNING id`, - name, - int64(10), - hash, - status, - 0, - ).Scan(&id); err != nil { - t.Fatalf("insert logical_file %q: %v", name, err) - } - return id - } - - completedID := insertLogical("logical-a", "hash-a", filestate.LogicalFileCompleted) - abortedID := insertLogical("logical-b", "hash-b", filestate.LogicalFileAborted) - - if _, err := dbconn.Exec( - `INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) VALUES ($1, $2, $3)`, - "/data/a-one.txt", - completedID, - 1, - ); err != nil { - t.Fatalf("insert first physical_file row: %v", err) - } - if _, err := dbconn.Exec( - `INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) VALUES ($1, $2, $3)`, - "/data/a-two.txt", - completedID, - 1, - ); err != nil { - t.Fatalf("insert second physical_file row: %v", err) - } - if _, err := dbconn.Exec( - `INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) VALUES ($1, $2, $3)`, - "/data/aborted.txt", - abortedID, - 1, - ); err != nil { - t.Fatalf("insert aborted physical_file row: %v", err) - } - - records, err := ListFilesResultWithDB(dbconn, nil) - if err != nil { - t.Fatalf("list files: %v", err) - } - if len(records) != 2 { - t.Fatalf("expected 2 current-state records for completed logical_file only, got %d", len(records)) - } - - paths := map[string]bool{} - for _, rec := range records { - paths[rec.Name] = true - if rec.ID != completedID { - t.Fatalf("expected listed row logical id %d, got %d", completedID, rec.ID) - } - } - - for _, expectedPath := range []string{"/data/a-one.txt", "/data/a-two.txt"} { - if !paths[expectedPath] { - t.Fatalf("missing listed current-state path %q in %+v", expectedPath, paths) - } - } - - limited, err := ListFilesResultWithDB(dbconn, []string{"--limit", "1", "--offset", "1"}) - if err != nil { - t.Fatalf("list files with pagination: %v", err) - } - if len(limited) != 1 { - t.Fatalf("expected one paginated result, got %d", len(limited)) - } - if !strings.HasPrefix(limited[0].Name, "/data/a-") { - t.Fatalf("unexpected paginated path: %s", limited[0].Name) - } - -} diff --git a/internal/listing/query.go b/internal/listing/query.go deleted file mode 100644 index a7780532..00000000 --- a/internal/listing/query.go +++ /dev/null @@ -1,68 +0,0 @@ -package listing - -import ( - "fmt" - "strconv" -) - -const MaxPaginationLimit int64 = 10000 - -func parseNonNegativeIntArg(flagName, value string) (int64, error) { - parsedValue, err := strconv.ParseInt(value, 10, 64) - if err != nil || parsedValue < 0 { - return 0, fmt.Errorf("invalid --%s value %q: must be a non-negative integer", flagName, value) - } - - return parsedValue, nil -} - -func parsePaginationArgs(args []string) (*int64, *int64, error) { - var limit *int64 - var offset *int64 - - for i := 0; i < len(args); i++ { - switch args[i] { - case "--limit": - if i+1 >= len(args) { - return nil, nil, fmt.Errorf("missing argument for --limit") - } - i++ - parsedValue, err := parseNonNegativeIntArg("limit", args[i]) - if err != nil { - return nil, nil, err - } - if parsedValue > MaxPaginationLimit { - return nil, nil, fmt.Errorf("invalid --limit value %q: must be <= %d", args[i], MaxPaginationLimit) - } - value := parsedValue - limit = &value - case "--offset": - if i+1 >= len(args) { - return nil, nil, fmt.Errorf("missing argument for --offset") - } - i++ - parsedValue, err := parseNonNegativeIntArg("offset", args[i]) - if err != nil { - return nil, nil, err - } - value := parsedValue - offset = &value - } - } - - return limit, offset, nil -} - -func applyPagination(query string, params []interface{}, paramIndex int, limit, offset *int64) (string, []interface{}) { - if limit != nil { - query += fmt.Sprintf(" LIMIT $%d", paramIndex) - params = append(params, *limit) - paramIndex++ - } - if offset != nil { - query += fmt.Sprintf(" OFFSET $%d", paramIndex) - params = append(params, *offset) - } - - return query, params -} diff --git a/internal/listing/query_test.go b/internal/listing/query_test.go deleted file mode 100644 index 1ecb000b..00000000 --- a/internal/listing/query_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package listing - -import ( - "strings" - "testing" -) - -func TestParsePaginationArgsDefaultsToNil(t *testing.T) { - limit, offset, err := parsePaginationArgs(nil) - if err != nil { - t.Fatalf("parse pagination args: %v", err) - } - if limit != nil { - t.Fatalf("expected nil limit, got %v", *limit) - } - if offset != nil { - t.Fatalf("expected nil offset, got %v", *offset) - } -} - -func TestParsePaginationArgsUsesLastValue(t *testing.T) { - limit, offset, err := parsePaginationArgs([]string{"--limit", "10", "--offset", "5", "--limit", "25"}) - if err != nil { - t.Fatalf("parse pagination args: %v", err) - } - if limit == nil || *limit != 25 { - t.Fatalf("expected limit 25, got %v", limit) - } - if offset == nil || *offset != 5 { - t.Fatalf("expected offset 5, got %v", offset) - } -} - -func TestParsePaginationArgsRejectsNegativeValues(t *testing.T) { - _, _, err := parsePaginationArgs([]string{"--offset", "-1"}) - if err == nil || !strings.Contains(err.Error(), "invalid --offset") { - t.Fatalf("expected error containing \"invalid --offset\", got: %v", err) - } -} - -func TestParsePaginationArgsRejectsLimitAboveMaximum(t *testing.T) { - _, _, err := parsePaginationArgs([]string{"--limit", "10001"}) - if err == nil || !strings.Contains(err.Error(), "invalid --limit") { - t.Fatalf("expected error containing \"invalid --limit\", got: %v", err) - } -} - -func TestParsePaginationArgsRejectsMissingLimitValue(t *testing.T) { - _, _, err := parsePaginationArgs([]string{"--limit"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --limit") { - t.Fatalf("expected missing --limit argument contract, got: %v", err) - } -} - -func TestParsePaginationArgsRejectsMissingOffsetValue(t *testing.T) { - _, _, err := parsePaginationArgs([]string{"--offset"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --offset") { - t.Fatalf("expected missing --offset argument contract, got: %v", err) - } -} - -func TestParsePaginationArgsRejectsNonIntegerLimitValue(t *testing.T) { - _, _, err := parsePaginationArgs([]string{"--limit", "NaN"}) - if err == nil || !strings.Contains(err.Error(), "invalid --limit") { - t.Fatalf("expected invalid --limit parse contract, got: %v", err) - } -} - -func TestApplyPaginationAppendsLimitThenOffset(t *testing.T) { - limitValue := int64(10) - offsetValue := int64(5) - - query, params := applyPagination("SELECT id, original_name, total_size, file_hash, status, created_at FROM logical_file ORDER BY created_at DESC", []interface{}{"completed", "report"}, 3, &limitValue, &offsetValue) - - if !strings.Contains(query, "LIMIT $3 OFFSET $4") { - t.Fatalf("unexpected query: %s", query) - } - if len(params) != 4 { - t.Fatalf("expected 4 params, got %d", len(params)) - } - if got := params[2]; got != int64(10) { - t.Fatalf("expected limit param 10, got %v", got) - } - if got := params[3]; got != int64(5) { - t.Fatalf("expected offset param 5, got %v", got) - } -} - -func TestApplyPaginationWithLimitOnly(t *testing.T) { - limitValue := int64(7) - - query, params := applyPagination("SELECT id, original_name, total_size, file_hash, status, created_at FROM logical_file", []interface{}{"completed"}, 2, &limitValue, nil) - - if !strings.Contains(query, "LIMIT $2") || strings.Contains(query, "OFFSET") { - t.Fatalf("unexpected query for limit-only case: %s", query) - } - if len(params) != 2 { - t.Fatalf("expected 2 params, got %d", len(params)) - } - if got := params[1]; got != int64(7) { - t.Fatalf("expected limit param 7, got %v", got) - } -} - -func TestApplyPaginationWithOffsetOnly(t *testing.T) { - offsetValue := int64(12) - - query, params := applyPagination("SELECT id, original_name, total_size, file_hash, status, created_at FROM logical_file", []interface{}{"completed"}, 2, nil, &offsetValue) - - if strings.Contains(query, "LIMIT") || !strings.Contains(query, "OFFSET $2") { - t.Fatalf("unexpected query for offset-only case: %s", query) - } - if len(params) != 2 { - t.Fatalf("expected 2 params, got %d", len(params)) - } - if got := params[1]; got != int64(12) { - t.Fatalf("expected offset param 12, got %v", got) - } -} - -func TestApplyPaginationWithNoPaginationLeavesQueryUnchanged(t *testing.T) { - baseQuery := "SELECT id, original_name, total_size, file_hash, status, created_at FROM logical_file ORDER BY created_at DESC" - baseParams := []interface{}{"completed"} - - query, params := applyPagination(baseQuery, baseParams, 2, nil, nil) - - if query != baseQuery { - t.Fatalf("expected query unchanged, got: %s", query) - } - if len(params) != 1 || params[0] != "completed" { - t.Fatalf("expected params unchanged, got: %#v", params) - } -} diff --git a/internal/listing/search.go b/internal/listing/search.go deleted file mode 100644 index b2c4da73..00000000 --- a/internal/listing/search.go +++ /dev/null @@ -1,112 +0,0 @@ -package listing - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/franchoy/coldkeep/internal/db" - filestate "github.com/franchoy/coldkeep/internal/status" -) - -// SearchFilesResultWithDB returns matching records using a caller-managed DB connection. -func SearchFilesResultWithDB(dbconn *sql.DB, args []string) ([]FileRecord, error) { - if dbconn == nil { - return nil, fmt.Errorf("db connection is nil") - } - - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() - - limit, offset, err := parsePaginationArgs(args) - if err != nil { - return nil, err - } - - query := ` - SELECT lf.id, pf.path, lf.file_hash, lf.total_size, lf.created_at - FROM physical_file pf - JOIN logical_file lf ON lf.id = pf.logical_file_id - WHERE lf.status = $1 - ` - params := []interface{}{filestate.LogicalFileCompleted} - paramIndex := 2 - - for i := 0; i < len(args); i++ { - switch args[i] { - case "--name": - if i+1 >= len(args) { - return nil, fmt.Errorf("missing argument for --name") - } - i++ - query += fmt.Sprintf(" AND LOWER(pf.path) LIKE LOWER($%d)", paramIndex) - params = append(params, "%"+args[i]+"%") - paramIndex++ - - case "--min-size": - if i+1 >= len(args) { - return nil, fmt.Errorf("missing argument for --min-size") - } - i++ - size, err := parseNonNegativeIntArg("min-size", args[i]) - if err != nil { - return nil, err - } - query += fmt.Sprintf(" AND total_size >= $%d", paramIndex) - params = append(params, size) - paramIndex++ - - case "--max-size": - if i+1 >= len(args) { - return nil, fmt.Errorf("missing argument for --max-size") - } - i++ - size, err := parseNonNegativeIntArg("max-size", args[i]) - if err != nil { - return nil, err - } - query += fmt.Sprintf(" AND total_size <= $%d", paramIndex) - params = append(params, size) - paramIndex++ - - case "--limit", "--offset": - if i+1 >= len(args) { - return nil, fmt.Errorf("missing argument for %s", args[i]) - } - i++ - } - } - - query += " ORDER BY pf.path ASC" - query, params = applyPagination(query, params, paramIndex, limit, offset) - - rows, err := dbconn.QueryContext(ctx, query, params...) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var records []FileRecord - for rows.Next() { - var id int64 - var name string - var fileHash string - var size int64 - var created time.Time - if err := rows.Scan(&id, &name, &fileHash, &size, &created); err != nil { - return nil, err - } - records = append(records, FileRecord{ - ID: id, - Name: name, - FileHash: fileHash, - SizeBytes: size, - CreatedAt: created.Format("2006-01-02 15:04:05"), - }) - } - if err := rows.Err(); err != nil { - return nil, err - } - return records, nil -} diff --git a/internal/listing/search_test.go b/internal/listing/search_test.go deleted file mode 100644 index eab5a317..00000000 --- a/internal/listing/search_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package listing - -import ( - "database/sql" - "strings" - "testing" - - "github.com/franchoy/coldkeep/internal/db" - filestate "github.com/franchoy/coldkeep/internal/status" - - _ "github.com/mattn/go-sqlite3" -) - -// openTestDB returns a non-nil *sql.DB backed by an in-memory SQLite connection. -// The listing package checks db != nil before any query, so we only need a valid -// handle — no schema migrations are required for argument-parsing tests. -func openTestDB(t *testing.T) *sql.DB { - t.Helper() - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("open test db: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func TestSearchFilesFailsWhenDBIsNil(t *testing.T) { - _, err := SearchFilesResultWithDB(nil, nil) - if err == nil || !strings.Contains(err.Error(), "db connection is nil") { - t.Fatalf("expected db-nil error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenNameArgIsMissing(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--name"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --name") { - t.Fatalf("expected missing-name error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenMinSizeArgIsMissing(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--min-size"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --min-size") { - t.Fatalf("expected missing-min-size error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenMaxSizeArgIsMissing(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--max-size"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --max-size") { - t.Fatalf("expected missing-max-size error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenLimitArgIsMissing(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--limit"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --limit") { - t.Fatalf("expected missing-limit error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenOffsetArgIsMissing(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--offset"}) - if err == nil || !strings.Contains(err.Error(), "missing argument for --offset") { - t.Fatalf("expected missing-offset error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenMinSizeArgIsInvalid(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--min-size", "-1"}) - if err == nil || !strings.Contains(err.Error(), "invalid --min-size value") { - t.Fatalf("expected invalid-min-size error contract, got: %v", err) - } -} - -func TestSearchFilesFailsWhenMaxSizeArgIsInvalid(t *testing.T) { - _, err := SearchFilesResultWithDB(openTestDB(t), []string{"--max-size", "not-a-number"}) - if err == nil || !strings.Contains(err.Error(), "invalid --max-size value") { - t.Fatalf("expected invalid-max-size error contract, got: %v", err) - } -} - -func TestSearchFilesUsesCurrentStatePhysicalPaths(t *testing.T) { - dbconn := openTestDB(t) - if err := db.RunMigrations(dbconn); err != nil { - t.Fatalf("run migrations: %v", err) - } - - insertLogical := func(name, hash string, size int64, status string) int64 { - t.Helper() - var id int64 - if err := dbconn.QueryRow( - `INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) - VALUES ($1, $2, $3, $4, $5, 'v1-simple-rolling') - RETURNING id`, - name, - size, - hash, - status, - 0, - ).Scan(&id); err != nil { - t.Fatalf("insert logical_file %q: %v", name, err) - } - return id - } - - idReport := insertLogical("report-logical", "hash-report", 120, filestate.LogicalFileCompleted) - idOther := insertLogical("other-logical", "hash-other", 42, filestate.LogicalFileCompleted) - _ = insertLogical("aborted-logical", "hash-aborted", 90, filestate.LogicalFileAborted) - - seed := []struct { - path string - id int64 - }{ - {path: "/home/u/docs/monthly-report.txt", id: idReport}, - {path: "/tmp/archive/report-copy.txt", id: idReport}, - {path: "/home/u/docs/notes.txt", id: idOther}, - } - for _, row := range seed { - if _, err := dbconn.Exec( - `INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) VALUES ($1, $2, $3)`, - row.path, - row.id, - 1, - ); err != nil { - t.Fatalf("insert physical_file %q: %v", row.path, err) - } - } - - records, err := SearchFilesResultWithDB(dbconn, []string{"--name", "report"}) - if err != nil { - t.Fatalf("search by name/path term: %v", err) - } - if len(records) != 2 { - t.Fatalf("expected 2 report path matches, got %d", len(records)) - } - for _, rec := range records { - if !strings.Contains(strings.ToLower(rec.Name), "report") { - t.Fatalf("expected record path to include report, got %q", rec.Name) - } - if rec.ID != idReport { - t.Fatalf("expected report logical ID %d, got %d", idReport, rec.ID) - } - } - - bySize, err := SearchFilesResultWithDB(dbconn, []string{"--min-size", "100"}) - if err != nil { - t.Fatalf("search by min-size: %v", err) - } - if len(bySize) != 2 { - t.Fatalf("expected 2 rows for logical size>=100 (same logical across two paths), got %d", len(bySize)) - } - - exactPath, err := SearchFilesResultWithDB(dbconn, []string{"--name", "/home/u/docs/monthly-report.txt"}) - if err != nil { - t.Fatalf("search by exact stored path token: %v", err) - } - if len(exactPath) != 1 || exactPath[0].Name != "/home/u/docs/monthly-report.txt" { - t.Fatalf("expected exact path match result, got %+v", exactPath) - } - -} diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index b3b54af7..c110cb90 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -23,8 +23,8 @@ import ( "github.com/franchoy/coldkeep/internal/chunk" "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/execution" - "github.com/franchoy/coldkeep/internal/listing" "github.com/franchoy/coldkeep/internal/maintenance" "github.com/franchoy/coldkeep/internal/recovery" "github.com/franchoy/coldkeep/internal/snapshot" @@ -13645,10 +13645,15 @@ func TestSearchListConsistencyWithFilters(t *testing.T) { stored[id] = f } - listRows, err := listing.ListFilesResultWithDB(dbconn, nil) + eng, err := engine.New(engine.Config{DB: dbconn, ContainerDir: container.ContainersDir, StoreContext: &sgctx}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + listResult, err := eng.ListFiles(context.Background(), engine.ListFilesRequest{}) if err != nil { t.Fatalf("list files: %v", err) } + listRows := listResult.Files if len(listRows) != len(files) { t.Fatalf("list row count mismatch: want %d got %d", len(files), len(listRows)) } @@ -13658,10 +13663,11 @@ func TestSearchListConsistencyWithFilters(t *testing.T) { listIDs[row.ID] = struct{}{} } - searchAlpha, err := listing.SearchFilesResultWithDB(dbconn, []string{"--name", "alpha"}) + searchAlphaResult, err := eng.SearchFiles(context.Background(), engine.SearchFilesRequest{NameContains: []string{"alpha"}}) if err != nil { t.Fatalf("search --name alpha: %v", err) } + searchAlpha := searchAlphaResult.Files if len(searchAlpha) != 3 { t.Fatalf("expected 3 alpha search rows, got %d", len(searchAlpha)) } @@ -13674,10 +13680,14 @@ func TestSearchListConsistencyWithFilters(t *testing.T) { } } - searchSized, err := listing.SearchFilesResultWithDB(dbconn, []string{"--min-size", "60000", "--max-size", "100000"}) + minSize, maxSize := int64(60000), int64(100000) + searchSizedResult, err := eng.SearchFiles(context.Background(), engine.SearchFilesRequest{ + MinSizeBytes: []int64{minSize}, MaxSizeBytes: []int64{maxSize}, + }) if err != nil { t.Fatalf("search by Size range: %v", err) } + searchSized := searchSizedResult.Files if len(searchSized) != 2 { t.Fatalf("expected 2 files in Size range, got %d", len(searchSized)) } From e1d5e535f9c593a800f9677cf2b17f9af6571a4e Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 14:12:48 +0200 Subject: [PATCH 13/27] Complete v1.13.12 configuration ownership --- .../configuration_engine_routing_test.go | 94 ++++++++ cmd/coldkeep/main.go | 207 ++++-------------- cmd/coldkeep/main_test.go | 43 ---- cmd/coldkeep/stored_path_engine_adapters.go | 4 + .../stored_path_engine_routing_test.go | 16 ++ docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...1.13.12-phase12-configuration-ownership.md | 51 +++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 18 +- internal/catalog/backend_contract_test.go | 32 +++ internal/catalog/catalog.go | 7 + internal/catalog/repository_configuration.go | 68 ++++++ internal/catalog/service.go | 21 +- internal/catalog/types.go | 14 ++ internal/engine/candidates.go | 30 +++ internal/engine/candidates_test.go | 2 +- internal/engine/contracts_test.go | 4 + internal/engine/default_engine.go | 4 + internal/engine/engine.go | 6 + internal/engine/repository_configuration.go | 163 ++++++++++++++ .../engine/repository_configuration_test.go | 104 +++++++++ 21 files changed, 679 insertions(+), 221 deletions(-) create mode 100644 cmd/coldkeep/configuration_engine_routing_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase12-configuration-ownership.md create mode 100644 internal/catalog/repository_configuration.go create mode 100644 internal/engine/repository_configuration.go create mode 100644 internal/engine/repository_configuration_test.go diff --git a/cmd/coldkeep/configuration_engine_routing_test.go b/cmd/coldkeep/configuration_engine_routing_test.go new file mode 100644 index 00000000..336edd09 --- /dev/null +++ b/cmd/coldkeep/configuration_engine_routing_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "database/sql" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" +) + +func TestRunConfigGetUsesEngineJSONParity(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + level := int64(5) + installConfigurationCommandStubs(t, dbconn, stubCommandEngine{ + getConfigurationFunc: func(_ context.Context, req engine.GetConfigurationRequest) (engine.GetConfigurationResult, error) { + if req.Key != engine.ConfigurationCompressionLevel { + t.Fatalf("unexpected configuration key: %s", req.Key) + } + return engine.GetConfigurationResult{Key: req.Key, Value: "5", IntegerValue: &level}, nil + }, + }) + + output := captureStdout(t, func() { + if err := runConfigCommand(parsedCommandLine{ + method: "config", positionals: []string{"get", "compression-level"}, + }, outputModeJSON); err != nil { + t.Fatalf("runConfigCommand: %v", err) + } + }) + if strings.TrimSpace(output) != `{"command":"config get","data":{"key":"compression-level","value":5},"status":"ok"}` { + t.Fatalf("unexpected config get JSON: %s", output) + } +} + +func TestRunConfigSetUsesEngineAndChangedProjection(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installConfigurationCommandStubs(t, dbconn, stubCommandEngine{ + setConfigurationFunc: func(_ context.Context, req engine.SetConfigurationRequest) (engine.SetConfigurationResult, error) { + if req.Key != engine.ConfigurationCompression || req.Value != " zstd " { + t.Fatalf("unexpected SetConfiguration request: %+v", req) + } + return engine.SetConfigurationResult{Key: req.Key, Value: "zstd", Changed: true}, nil + }, + }) + + output := captureStdout(t, func() { + if err := runConfigCommand(parsedCommandLine{ + method: "config", positionals: []string{"set", "compression", " zstd "}, + }, outputModeText); err != nil { + t.Fatalf("runConfigCommand: %v", err) + } + }) + for _, want := range []string{"compression set to zstd", "This affects only NEW blocks", "Blocks remain readable"} { + if !strings.Contains(output, want) { + t.Fatalf("expected config set output to contain %q, got:\n%s", want, output) + } + } +} + +func TestRunConfigSetMapsEngineValidationToUsage(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installConfigurationCommandStubs(t, dbconn, stubCommandEngine{ + setConfigurationFunc: func(context.Context, engine.SetConfigurationRequest) (engine.SetConfigurationResult, error) { + return engine.SetConfigurationResult{}, engine.NewError( + engine.ErrorInvalidArgument, "set configuration", + "invalid compression codec \"xz\", must be 'none' or 'zstd'", "", nil, + ) + }, + }) + err := runConfigCommand(parsedCommandLine{ + method: "config", positionals: []string{"set", "compression", "xz"}, + }, outputModeText) + if err == nil || classifyExitCode(err) != exitUsage || !strings.Contains(err.Error(), "invalid compression codec") { + t.Fatalf("expected usage-class engine validation, got %v", err) + } +} + +func installConfigurationCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { + t.Helper() + originalLoad := loadDefaultStorageContextPhase + originalNewEngine := newConfigurationCommandEngine + t.Cleanup(func() { + loadDefaultStorageContextPhase = originalLoad + newConfigurationCommandEngine = originalNewEngine + }) + loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { + return storage.StorageContext{DB: dbconn}, nil + } + newConfigurationCommandEngine = func(storage.StorageContext) (engine.Engine, error) { + return stub, nil + } +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index f1221f2c..c0144b5e 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -556,10 +556,6 @@ var verifySummaryPhase = func(dbconn *sql.DB, target string, fileID int64) (veri var runChunkerBenchmarkPhase = runChunkerBenchmark var runCoreBenchmarkPhase = runCoreBenchmark var runBenchmarkDeterminismPhase = validateBenchmarkDeterminism -var isDeprecatedChunkerVersionPhase = func(v chunk.Version) (bool, string) { - // Future-proof policy hook: no deprecated chunkers currently. - return false, "" -} type cliError struct { code int @@ -1257,24 +1253,6 @@ func runVersionCommand(mode cliOutputMode) error { return nil } -func validateConfigDefaultChunkerVersion(raw string) (chunk.Version, error) { - v := chunk.Version(strings.TrimSpace(raw)) - if !chunk.IsWellFormedVersion(v) { - return "", usageErrorf("invalid default-chunker value %q: malformed version", raw) - } - if _, ok := chunk.DefaultRegistry().Get(v); !ok { - return "", usageErrorf("invalid default-chunker value %q: unknown chunker version", raw) - } - if deprecated, reason := isDeprecatedChunkerVersionPhase(v); deprecated { - reason = strings.TrimSpace(reason) - if reason == "" { - return "", usageErrorf("invalid default-chunker value %q: deprecated chunker version", raw) - } - return "", usageErrorf("invalid default-chunker value %q: deprecated chunker version (%s)", raw, reason) - } - return v, nil -} - func runConfigCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { if err := ensureAllowedFlags(parsed, "output", "json"); err != nil { return err @@ -1307,67 +1285,25 @@ func runConfigCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return fmt.Errorf("load storage context: %w", err) } defer func() { _ = sgctx.Close() }() - - repo := storage.NewRepository(sgctx.DB) + eng, err := newConfigurationCommandEngine(sgctx) + if err != nil { + return err + } + engineKey := engine.ConfigurationKey(key) switch subcommand { case "get": - switch key { - case "compression": - codec, err := repo.GetDefaultCompression(sgctx.DB) - if err != nil { - return err - } - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "config get", - "data": map[string]any{ - "key": "compression", - "value": codec, - }, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil - } - _, _ = fmt.Fprintln(os.Stdout, codec) - return nil - - case "compression-level": - level, err := repo.GetDefaultCompressionLevel(sgctx.DB) - if err != nil { - return err - } - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "config get", - "data": map[string]any{ - "key": "compression-level", - "value": level, - }, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil - } - _, _ = fmt.Fprintf(os.Stdout, "%d\n", level) - return nil - } - - v, err := repo.GetDefaultChunkerVersion() + result, err := eng.GetConfiguration(context.Background(), engine.GetConfigurationRequest{Key: engineKey}) if err != nil { return err } - if outputMode == outputModeJSON { payload := map[string]any{ "status": "ok", "command": "config get", "data": map[string]any{ - "key": "default-chunker", - "value": string(v), + "key": key, + "value": configurationResultValue(result.Value, result.IntegerValue), }, } encoded, _ := json.Marshal(payload) @@ -1375,100 +1311,26 @@ func runConfigCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return nil } - _, _ = fmt.Fprintln(os.Stdout, string(v)) + _, _ = fmt.Fprintln(os.Stdout, result.Value) return nil case "set": - switch key { - case "compression": - codec := strings.TrimSpace(parsed.positionals[2]) - if !storage.IsRegisteredCompressionCodec(codec) { - return usageErrorf("invalid compression codec %q, must be 'none' or 'zstd'", codec) - } - previous, err := repo.GetDefaultCompression(sgctx.DB) - if err != nil { - return err - } - if err := repo.SetDefaultCompression(sgctx.DB, codec); err != nil { - return err - } - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "config set", - "data": map[string]any{ - "key": "compression", - "value": codec, - }, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil - } - _, _ = fmt.Fprintf(os.Stdout, "compression set to %s\n", codec) - if previous != codec { - _, _ = fmt.Fprintln(os.Stdout, "ℹ️ This affects only NEW blocks. Existing blocks are not recompressed.") - _, _ = fmt.Fprintln(os.Stdout, " Blocks remain readable according to their stored metadata.") - } - return nil - - case "compression-level": - levelStr := strings.TrimSpace(parsed.positionals[2]) - level, err := strconv.Atoi(levelStr) - if err != nil { - return usageErrorf("invalid compression-level %q, must be an integer 1-9", levelStr) - } - if level < 1 || level > 9 { - return usageErrorf("compression-level %d out of range, must be 1-9", level) - } - previous, err := repo.GetDefaultCompressionLevel(sgctx.DB) - if err != nil { - return err - } - if err := repo.SetDefaultCompressionLevel(sgctx.DB, level); err != nil { - return err - } - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "config set", - "data": map[string]any{ - "key": "compression-level", - "value": level, - }, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil - } - _, _ = fmt.Fprintf(os.Stdout, "compression-level set to %d\n", level) - if previous != level { - _, _ = fmt.Fprintln(os.Stdout, "ℹ️ This affects only NEW blocks. Existing blocks are not recompressed.") - } - return nil - } - - v, err := validateConfigDefaultChunkerVersion(parsed.positionals[2]) - if err != nil { - return err - } - - previous, err := repo.GetDefaultChunkerVersion() + result, err := eng.SetConfiguration(context.Background(), engine.SetConfigurationRequest{ + Key: engineKey, Value: parsed.positionals[2], + }) if err != nil { + if engine.IsCode(err, engine.ErrorInvalidArgument) { + return usageErrorf("%s", err) + } return err } - - if err := repo.SetDefaultChunkerVersion(v); err != nil { - return err - } - if outputMode == outputModeJSON { payload := map[string]any{ "status": "ok", "command": "config set", "data": map[string]any{ - "key": "default-chunker", - "value": string(v), + "key": key, + "value": configurationResultValue(result.Value, result.IntegerValue), }, } encoded, _ := json.Marshal(payload) @@ -1476,11 +1338,7 @@ func runConfigCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return nil } - _, _ = fmt.Fprintf(os.Stdout, "default-chunker set to %s\n", v) - if previous != v { - _, _ = fmt.Fprintln(os.Stdout, "Warning: This affects only new stored data.") - _, _ = fmt.Fprintln(os.Stdout, "Existing data remains unchanged.") - } + emitConfigurationSetText(key, result) return nil default: @@ -1488,6 +1346,35 @@ func runConfigCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } } +func configurationResultValue(value string, integerValue *int64) any { + if integerValue != nil { + return *integerValue + } + return value +} + +func emitConfigurationSetText(key string, result engine.SetConfigurationResult) { + switch key { + case "compression": + _, _ = fmt.Fprintf(os.Stdout, "compression set to %s\n", result.Value) + if result.Changed { + _, _ = fmt.Fprintln(os.Stdout, "ℹ️ This affects only NEW blocks. Existing blocks are not recompressed.") + _, _ = fmt.Fprintln(os.Stdout, " Blocks remain readable according to their stored metadata.") + } + case "compression-level": + _, _ = fmt.Fprintf(os.Stdout, "compression-level set to %s\n", result.Value) + if result.Changed { + _, _ = fmt.Fprintln(os.Stdout, "ℹ️ This affects only NEW blocks. Existing blocks are not recompressed.") + } + case "default-chunker": + _, _ = fmt.Fprintf(os.Stdout, "default-chunker set to %s\n", result.Value) + if result.Changed { + _, _ = fmt.Fprintln(os.Stdout, "Warning: This affects only new stored data.") + _, _ = fmt.Fprintln(os.Stdout, "Existing data remains unchanged.") + } + } +} + func verifyLevelToString(level verify.VerifyLevel) string { switch level { case verify.VerifyFast: diff --git a/cmd/coldkeep/main_test.go b/cmd/coldkeep/main_test.go index 53c3863a..9dc53521 100644 --- a/cmd/coldkeep/main_test.go +++ b/cmd/coldkeep/main_test.go @@ -4727,49 +4727,6 @@ func TestRunConfigCommandSetRejectsUnknownVersion(t *testing.T) { } } -func TestRunConfigCommandSetRejectsDeprecatedVersion(t *testing.T) { - originalLoad := loadDefaultStorageContextPhase - originalDeprecation := isDeprecatedChunkerVersionPhase - t.Cleanup(func() { - loadDefaultStorageContextPhase = originalLoad - isDeprecatedChunkerVersionPhase = originalDeprecation - }) - - dbPath := filepath.Join(t.TempDir(), "config_deprecated.sqlite") - loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { - dbconn, err := sql.Open("sqlite3", dbPath) - if err != nil { - return storage.StorageContext{}, err - } - if err := dbpkg.RunMigrations(dbconn); err != nil { - _ = dbconn.Close() - return storage.StorageContext{}, err - } - return storage.StorageContext{DB: dbconn}, nil - } - - isDeprecatedChunkerVersionPhase = func(v chunk.Version) (bool, string) { - if v == chunk.VersionV2FastCDC { - return true, "scheduled removal" - } - return false, "" - } - - err := runConfigCommand(parsedCommandLine{ - method: "config", - positionals: []string{"set", "default-chunker", string(chunk.VersionV2FastCDC)}, - }, outputModeText) - if err == nil || !strings.Contains(err.Error(), "deprecated chunker version") { - t.Fatalf("expected deprecated-version error, got: %v", err) - } - if !strings.Contains(err.Error(), "scheduled removal") { - t.Fatalf("expected deprecated-version reason in error, got: %v", err) - } - if got := classifyExitCode(err); got != exitUsage { - t.Fatalf("expected usage exit code %d, got %d", exitUsage, got) - } -} - func TestRunConfigCommandGetJSON(t *testing.T) { originalLoad := loadDefaultStorageContextPhase t.Cleanup(func() { diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index 16261242..b9a4b931 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -36,6 +36,10 @@ var newStoreFolderCommandEngine = func(sgctx storage.StorageContext) (engine.Eng }) } +var newConfigurationCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 91b7c2e5..2f0e91c1 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -25,6 +25,8 @@ type stubCommandEngine struct { storeFolderFunc func(context.Context, engine.StoreFolderRequest) (engine.StoreFolderResult, error) listFilesFunc func(context.Context, engine.ListFilesRequest) (engine.ListFilesResult, error) searchFilesFunc func(context.Context, engine.SearchFilesRequest) (engine.SearchFilesResult, error) + getConfigurationFunc func(context.Context, engine.GetConfigurationRequest) (engine.GetConfigurationResult, error) + setConfigurationFunc func(context.Context, engine.SetConfigurationRequest) (engine.SetConfigurationResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -83,6 +85,20 @@ func (s stubCommandEngine) SearchFiles(ctx context.Context, req engine.SearchFil return engine.SearchFilesResult{}, errors.New("unexpected SearchFiles call") } +func (s stubCommandEngine) GetConfiguration(ctx context.Context, req engine.GetConfigurationRequest) (engine.GetConfigurationResult, error) { + if s.getConfigurationFunc != nil { + return s.getConfigurationFunc(ctx, req) + } + return engine.GetConfigurationResult{}, errors.New("unexpected GetConfiguration call") +} + +func (s stubCommandEngine) SetConfiguration(ctx context.Context, req engine.SetConfigurationRequest) (engine.SetConfigurationResult, error) { + if s.setConfigurationFunc != nil { + return s.setConfigurationFunc(ctx, req) + } + return engine.SetConfigurationResult{}, errors.New("unexpected SetConfiguration call") +} + func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 6ad4bb80..341c84dd 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -81,11 +81,13 @@ ## Phase 12 — Configuration Ownership -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 12 configuration ownership](v1.13.12-phase12-configuration-ownership.md). ## Phase 13 — Snapshot Read-Side Completion -**Status:** Not started +**Status:** Next ## Phase 14 — Inspect, Stats, and Verify Routing diff --git a/docs/release/v1.13/v1.13.12-phase12-configuration-ownership.md b/docs/release/v1.13/v1.13.12-phase12-configuration-ownership.md new file mode 100644 index 00000000..a60a0caa --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase12-configuration-ownership.md @@ -0,0 +1,51 @@ +# Coldkeep v1.13.12 Phase 12 — Configuration Ownership + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `12 — Configuration Ownership` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Catalog and engine ownership + +`RepositoryConfigurationCatalog` now owns typed `repository_config` reads and +upserts. It reports missing values, previous metadata, and changed/unchanged +state without exposing SQL or database handles. + +The engine maps the three public keys—`default-chunker`, `compression`, and +`compression-level`—to catalog keys. It owns legacy fallback defaults, +normalization, registered chunker checks, injectable deprecation policy, +registered compression checks, level parsing/range validation, and effective +changed/unchanged semantics. Compression levels retain an explicit integer +projection rather than an untyped value. + +## Production routing and compatibility + +The production config command performs pre-initialization syntax validation, +opens its existing configured context, invokes one typed get/set engine method, +and renders the result. It no longer constructs a storage repository or calls +storage configuration methods directly. + +Existing public keys, defaults, text, JSON value types, usage classification, +new-data-only warnings, unchanged-value warning suppression, metadata rows, and +write-only policy are preserved. No schema, migration, backend default, stored +payload, logical file, chunk, or historical block metadata changed. + +## Validation + +- Direct catalog and engine configuration suites: pass. +- CLI routing plus existing configuration compatibility/safety suites: pass. +- Shared SQLite/PostgreSQL repository-configuration contract: pass. +- Focused catalog, engine, and CLI race suites: pass. +- Full `go test -count=1 ./...`: pass. +- Validation root: `/tmp/coldkeep-v11312-phase12.18xw7S/`, containing isolated + caches, temporary files, storage roots, and SQLite databases. + +PostgreSQL validation created and dropped exactly +`coldkeep_bt_testcatalogcontractrepositoryconfiguration_2b8953_1`. A read-only +maintenance query confirmed that exact name is absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 13 is authorized to complete snapshot read-side engine ownership without +changing snapshot metadata, selector, filtering, or presentation semantics. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 256637c4..13044dfe 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–11 are Complete. Phase 12 is Next: activate typed catalog and engine -repository-configuration ownership while preserving keys, defaults, validation, -changed/unchanged semantics, metadata, and public output. +Phases 0–12 are Complete. Phase 13 is Next: complete snapshot list, tree, show, +stats, and every diff mode behind typed engine read operations with repeated +selectors and frozen filtering/output semantics. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 56011ede..be52a61c 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -186,11 +186,25 @@ ## Phase 12 — Configuration Ownership -**Phase status:** Next +**Phase status:** Complete + +- [x] Catalog owns typed repository configuration get/upsert truth with + present/missing and changed/unchanged metadata. +- [x] Engine owns public key mapping, fallback defaults, whitespace + normalization, chunker registration/deprecation, compression codec, and + compression-level validation. +- [x] `GetConfiguration` and `SetConfiguration` return neutral typed values; + compression-level preserves its numeric JSON projection. +- [x] Production config code no longer calls storage repository configuration + helpers and routes each operation through the engine. +- [x] CLI syntax, usage errors, text/JSON output, new-data-only warnings, and + unchanged-value warning suppression remain frozen. +- [x] Direct engine, CLI routing, read/write safety, race, and shared + SQLite/PostgreSQL catalog contracts pass. ## Phase 13 — Snapshot Read-Side Completion -**Phase status:** Not started +**Phase status:** Next ## Phase 14 — Inspect, Stats, and Verify Routing diff --git a/internal/catalog/backend_contract_test.go b/internal/catalog/backend_contract_test.go index 63bb0709..b9d6818b 100644 --- a/internal/catalog/backend_contract_test.go +++ b/internal/catalog/backend_contract_test.go @@ -226,6 +226,38 @@ func TestCatalogContractCurrentFileQueriesAcrossBackends(t *testing.T) { }) } +func TestCatalogContractRepositoryConfigurationAcrossBackends(t *testing.T) { + forEachCatalogBackend(t, func(t *testing.T, backend backendtest.Backend) { + svc := catalog.NewServiceFromSQL(backend.DB) + missing, err := svc.GetRepositoryConfiguration(context.Background(), "phase12_missing") + if err != nil || missing.Exists || missing.Key != "phase12_missing" || missing.Value != "" { + t.Fatalf("missing repository configuration: got (%+v, %v)", missing, err) + } + + first, err := svc.SetRepositoryConfiguration(context.Background(), "compression", "zstd") + if err != nil || !first.PreviouslySet || !first.Changed || first.Value != "zstd" || first.PreviousValue != "none" { + t.Fatalf("first repository configuration set: got (%+v, %v)", first, err) + } + second, err := svc.SetRepositoryConfiguration(context.Background(), "compression", "zstd") + if err != nil || !second.PreviouslySet || second.Changed || second.PreviousValue != "zstd" { + t.Fatalf("unchanged repository configuration set: got (%+v, %v)", second, err) + } + got, err := svc.GetRepositoryConfiguration(context.Background(), "compression") + if err != nil || !got.Exists || got.Value != "zstd" { + t.Fatalf("repository configuration round trip: got (%+v, %v)", got, err) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := svc.GetRepositoryConfiguration(cancelled, "compression"); !catalog.IsCode(err, catalog.ErrorCancelled) { + t.Fatalf("cancelled repository configuration get: %v", err) + } + if _, err := svc.SetRepositoryConfiguration(context.Background(), " ", "x"); !catalog.IsCode(err, catalog.ErrorInvalidArgument) { + t.Fatalf("blank repository configuration key: %v", err) + } + }) +} + func requirePhysicalFiles(t *testing.T, svc interface { FindPhysicalFilesForLogicalFile(context.Context, int64) ([]catalog.PhysicalFileRef, error) }, id int64) []catalog.PhysicalFileRef { diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 9199504a..9b5e32e3 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -74,6 +74,12 @@ type CurrentFileCatalog interface { SearchCurrentFiles(ctx context.Context, filter CurrentFileSearch) ([]CurrentFileRef, error) } +// RepositoryConfigurationCatalog owns repository_config metadata truth. +type RepositoryConfigurationCatalog interface { + GetRepositoryConfiguration(ctx context.Context, key string) (RepositoryConfigurationRef, error) + SetRepositoryConfiguration(ctx context.Context, key, value string) (SetRepositoryConfigurationResult, error) +} + // SnapshotCatalog exposes snapshot metadata. type SnapshotCatalog interface { // FindSnapshot returns the snapshot with the given ID. @@ -141,6 +147,7 @@ type Catalog interface { LogicalFileCatalog PhysicalFileCatalog CurrentFileCatalog + RepositoryConfigurationCatalog SnapshotCatalog SnapshotGraphCatalog ReachabilityCatalog diff --git a/internal/catalog/repository_configuration.go b/internal/catalog/repository_configuration.go new file mode 100644 index 00000000..6e0c2988 --- /dev/null +++ b/internal/catalog/repository_configuration.go @@ -0,0 +1,68 @@ +package catalog + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +func (s *Service) GetRepositoryConfiguration(ctx context.Context, key string) (RepositoryConfigurationRef, error) { + if ctx == nil { + ctx = context.Background() + } + key = strings.TrimSpace(key) + if key == "" { + return RepositoryConfigurationRef{}, NewError(ErrorInvalidArgument, "get repository configuration", "configuration_key_nonblank", "configuration key is required", nil) + } + if err := ctx.Err(); err != nil { + return RepositoryConfigurationRef{}, repositoryConfigurationError("get repository configuration", err) + } + ref := RepositoryConfigurationRef{Key: key} + err := s.db.QueryRowContext(ctx, `SELECT value FROM repository_config WHERE key = $1`, key).Scan(&ref.Value) + if errors.Is(err, sql.ErrNoRows) { + return ref, nil + } + if err != nil { + return RepositoryConfigurationRef{}, repositoryConfigurationError("get repository configuration", err) + } + ref.Exists = true + return ref, nil +} + +func (s *Service) SetRepositoryConfiguration(ctx context.Context, key, value string) (SetRepositoryConfigurationResult, error) { + if ctx == nil { + ctx = context.Background() + } + key = strings.TrimSpace(key) + if key == "" { + return SetRepositoryConfigurationResult{}, NewError(ErrorInvalidArgument, "set repository configuration", "configuration_key_nonblank", "configuration key is required", nil) + } + if err := ctx.Err(); err != nil { + return SetRepositoryConfigurationResult{}, repositoryConfigurationError("set repository configuration", err) + } + previous, err := s.GetRepositoryConfiguration(ctx, key) + if err != nil { + return SetRepositoryConfigurationResult{}, err + } + result := SetRepositoryConfigurationResult{ + Key: key, Value: value, PreviousValue: previous.Value, + PreviouslySet: previous.Exists, Changed: !previous.Exists || previous.Value != value, + } + if _, err := s.db.ExecContext(ctx, ` +INSERT INTO repository_config(key, value) +VALUES($1, $2) +ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, value); err != nil { + return SetRepositoryConfigurationResult{}, repositoryConfigurationError("set repository configuration", err) + } + return result, nil +} + +func repositoryConfigurationError(operation string, err error) error { + code := ErrorOperationFailed + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + code = ErrorCancelled + } + return NewError(code, operation, "", fmt.Sprintf("%s failed", operation), err) +} diff --git a/internal/catalog/service.go b/internal/catalog/service.go index bb41e45c..8ca7d9fe 100644 --- a/internal/catalog/service.go +++ b/internal/catalog/service.go @@ -13,16 +13,17 @@ type Service struct { // compile-time interface assertions — all sub-interfaces must remain satisfied. var ( - _ Catalog = (*Service)(nil) - _ LogicalFileCatalog = (*Service)(nil) - _ PhysicalFileCatalog = (*Service)(nil) - _ CurrentFileCatalog = (*Service)(nil) - _ SnapshotCatalog = (*Service)(nil) - _ SnapshotGraphCatalog = (*Service)(nil) - _ ReachabilityCatalog = (*Service)(nil) - _ PlacementCatalog = (*Service)(nil) - _ RestorePlanCatalog = (*Service)(nil) - _ GCPlanCatalog = (*Service)(nil) + _ Catalog = (*Service)(nil) + _ LogicalFileCatalog = (*Service)(nil) + _ PhysicalFileCatalog = (*Service)(nil) + _ CurrentFileCatalog = (*Service)(nil) + _ RepositoryConfigurationCatalog = (*Service)(nil) + _ SnapshotCatalog = (*Service)(nil) + _ SnapshotGraphCatalog = (*Service)(nil) + _ ReachabilityCatalog = (*Service)(nil) + _ PlacementCatalog = (*Service)(nil) + _ RestorePlanCatalog = (*Service)(nil) + _ GCPlanCatalog = (*Service)(nil) ) // NewService constructs a Service backed by the given DB. Both *sql.DB and diff --git a/internal/catalog/types.go b/internal/catalog/types.go index 093d8f5e..3094c5ae 100644 --- a/internal/catalog/types.go +++ b/internal/catalog/types.go @@ -47,6 +47,20 @@ type CurrentFileSearch struct { Page CurrentFilePage } +type RepositoryConfigurationRef struct { + Key string + Value string + Exists bool +} + +type SetRepositoryConfigurationResult struct { + Key string + Value string + PreviousValue string + PreviouslySet bool + Changed bool +} + type SnapshotRef struct { ID string Type string diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index fb88b20e..0258fd01 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -192,6 +192,36 @@ type SearchFilesResult struct { Files []CurrentFile } +type ConfigurationKey string + +const ( + ConfigurationDefaultChunker ConfigurationKey = "default-chunker" + ConfigurationCompression ConfigurationKey = "compression" + ConfigurationCompressionLevel ConfigurationKey = "compression-level" +) + +type GetConfigurationRequest struct { + Key ConfigurationKey +} + +type GetConfigurationResult struct { + Key ConfigurationKey + Value string + IntegerValue *int64 +} + +type SetConfigurationRequest struct { + Key ConfigurationKey + Value string +} + +type SetConfigurationResult struct { + Key ConfigurationKey + Value string + IntegerValue *int64 + Changed bool +} + // --------------------------------------------------------------------------- // Restore // --------------------------------------------------------------------------- diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index 3ae9c9ea..008da7e5 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 16cd2d69..de14c2aa 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -49,6 +49,10 @@ func activeCoreContractTypes() []struct { {"ListFilesResult", engine.ListFilesResult{}}, {"SearchFilesRequest", engine.SearchFilesRequest{}}, {"SearchFilesResult", engine.SearchFilesResult{}}, + {"GetConfigurationRequest", engine.GetConfigurationRequest{}}, + {"GetConfigurationResult", engine.GetConfigurationResult{}}, + {"SetConfigurationRequest", engine.SetConfigurationRequest{}}, + {"SetConfigurationResult", engine.SetConfigurationResult{}}, {"RestoreRequest", engine.RestoreRequest{}}, {"RestoreItemResult", engine.RestoreItemResult{}}, {"RestoreResult", engine.RestoreResult{}}, diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index 75f3e790..3fd76730 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/franchoy/coldkeep/internal/catalog" + "github.com/franchoy/coldkeep/internal/chunk" "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/maintenance" "github.com/franchoy/coldkeep/internal/observability" @@ -36,6 +37,9 @@ type Config struct { // StoreContext provides writer+chunker-aware dependencies for active store // orchestration. StoreContext *storage.StorageContext + // ChunkerDeprecationPolicy optionally rejects registered chunkers for new + // repository defaults. Nil means no registered chunker is deprecated. + ChunkerDeprecationPolicy func(chunk.Version) (bool, string) } // DefaultEngine is the canonical Engine implementation. diff --git a/internal/engine/engine.go b/internal/engine/engine.go index cedfb5f3..f99895e3 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -90,6 +90,12 @@ type Engine interface { // SearchFiles returns completed current-state paths matching typed filters. SearchFiles(ctx context.Context, req SearchFilesRequest) (SearchFilesResult, error) + // GetConfiguration returns one validated repository write default. + GetConfiguration(ctx context.Context, req GetConfigurationRequest) (GetConfigurationResult, error) + + // SetConfiguration validates and persists one repository write default. + SetConfiguration(ctx context.Context, req SetConfigurationRequest) (SetConfigurationResult, error) + // Remove removes logical files from the repository by logical file ID. // Safety invariant: Remove must never make valid data unrecoverable. // Method selection owns addressing semantics: this method is by-ID only. diff --git a/internal/engine/repository_configuration.go b/internal/engine/repository_configuration.go new file mode 100644 index 00000000..83de2f61 --- /dev/null +++ b/internal/engine/repository_configuration.go @@ -0,0 +1,163 @@ +package engine + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/franchoy/coldkeep/internal/catalog" + "github.com/franchoy/coldkeep/internal/chunk" + "github.com/franchoy/coldkeep/internal/storage" +) + +const ( + catalogDefaultChunkerKey = "default_chunker" + catalogCompressionKey = "compression" + catalogCompressionLevelKey = "compression_level" + defaultConfigurationCodec = "none" + defaultConfigurationLevel = int64(3) + minimumCompressionLevel = int64(1) + maximumCompressionLevel = int64(9) +) + +func (e *DefaultEngine) GetConfiguration(ctx context.Context, req GetConfigurationRequest) (GetConfigurationResult, error) { + catalogKey, err := configurationCatalogKey(req.Key) + if err != nil { + return GetConfigurationResult{}, TranslateErrorAs("get configuration", ErrorInvalidArgument, err) + } + ref, err := catalog.NewServiceFromSQL(e.config.DB).GetRepositoryConfiguration(ctx, catalogKey) + if err != nil { + return GetConfigurationResult{}, TranslateError("get configuration", err) + } + value := ref.Value + if !ref.Exists { + value = configurationDefault(req.Key) + } + result, err := validatedConfigurationResult(req.Key, value) + if err != nil { + return GetConfigurationResult{}, TranslateError("get configuration", err) + } + return result, nil +} + +func (e *DefaultEngine) SetConfiguration(ctx context.Context, req SetConfigurationRequest) (SetConfigurationResult, error) { + catalogKey, err := configurationCatalogKey(req.Key) + if err != nil { + return SetConfigurationResult{}, TranslateErrorAs("set configuration", ErrorInvalidArgument, err) + } + normalized, integerValue, err := e.normalizeConfigurationValue(req.Key, req.Value) + if err != nil { + return SetConfigurationResult{}, TranslateErrorAs("set configuration", ErrorInvalidArgument, err) + } + previous, err := e.GetConfiguration(ctx, GetConfigurationRequest{Key: req.Key}) + if err != nil { + return SetConfigurationResult{}, err + } + if _, err := catalog.NewServiceFromSQL(e.config.DB).SetRepositoryConfiguration(ctx, catalogKey, normalized); err != nil { + return SetConfigurationResult{}, TranslateError("set configuration", err) + } + return SetConfigurationResult{ + Key: req.Key, Value: normalized, IntegerValue: integerValue, + Changed: previous.Value != normalized, + }, nil +} + +func configurationCatalogKey(key ConfigurationKey) (string, error) { + switch key { + case ConfigurationDefaultChunker: + return catalogDefaultChunkerKey, nil + case ConfigurationCompression: + return catalogCompressionKey, nil + case ConfigurationCompressionLevel: + return catalogCompressionLevelKey, nil + default: + return "", fmt.Errorf("unknown config key: %s", key) + } +} + +func configurationDefault(key ConfigurationKey) string { + switch key { + case ConfigurationDefaultChunker: + return string(chunk.DefaultChunkerVersion) + case ConfigurationCompression: + return defaultConfigurationCodec + case ConfigurationCompressionLevel: + return strconv.FormatInt(defaultConfigurationLevel, 10) + default: + return "" + } +} + +func validatedConfigurationResult(key ConfigurationKey, raw string) (GetConfigurationResult, error) { + value := strings.TrimSpace(raw) + result := GetConfigurationResult{Key: key, Value: value} + switch key { + case ConfigurationDefaultChunker: + version := chunk.Version(value) + if !chunk.IsWellFormedVersion(version) { + return GetConfigurationResult{}, fmt.Errorf("repository default chunker version %q is malformed", version) + } + if _, ok := chunk.DefaultRegistry().Get(version); !ok { + return GetConfigurationResult{}, fmt.Errorf("repository default chunker version validation failed: chunker version %q is not registered in this binary", version) + } + case ConfigurationCompression: + if !storage.IsRegisteredCompressionCodec(value) { + return GetConfigurationResult{}, fmt.Errorf("repository default compression codec %q is not registered", value) + } + case ConfigurationCompressionLevel: + var level int64 + if _, err := fmt.Sscanf(value, "%d", &level); err != nil { + return GetConfigurationResult{}, fmt.Errorf("repository default compression level %q is not a valid integer: %w", raw, err) + } + if level < minimumCompressionLevel || level > maximumCompressionLevel { + return GetConfigurationResult{}, fmt.Errorf("repository default compression level %d is out of range [%d, %d]", level, minimumCompressionLevel, maximumCompressionLevel) + } + result.Value = strconv.FormatInt(level, 10) + result.IntegerValue = int64Pointer(level) + } + return result, nil +} + +func (e *DefaultEngine) normalizeConfigurationValue(key ConfigurationKey, raw string) (string, *int64, error) { + value := strings.TrimSpace(raw) + switch key { + case ConfigurationDefaultChunker: + version := chunk.Version(value) + if !chunk.IsWellFormedVersion(version) { + return "", nil, fmt.Errorf("invalid default-chunker value %q: malformed version", raw) + } + if _, ok := chunk.DefaultRegistry().Get(version); !ok { + return "", nil, fmt.Errorf("invalid default-chunker value %q: unknown chunker version", raw) + } + if e.config.ChunkerDeprecationPolicy != nil { + if deprecated, reason := e.config.ChunkerDeprecationPolicy(version); deprecated { + reason = strings.TrimSpace(reason) + if reason == "" { + return "", nil, fmt.Errorf("invalid default-chunker value %q: deprecated chunker version", raw) + } + return "", nil, fmt.Errorf("invalid default-chunker value %q: deprecated chunker version (%s)", raw, reason) + } + } + return string(version), nil, nil + case ConfigurationCompression: + if !storage.IsRegisteredCompressionCodec(value) { + return "", nil, fmt.Errorf("invalid compression codec %q, must be 'none' or 'zstd'", value) + } + return value, nil, nil + case ConfigurationCompressionLevel: + level, err := strconv.Atoi(value) + if err != nil { + return "", nil, fmt.Errorf("invalid compression-level %q, must be an integer 1-9", value) + } + if level < int(minimumCompressionLevel) || level > int(maximumCompressionLevel) { + return "", nil, fmt.Errorf("compression-level %d out of range, must be 1-9", level) + } + numeric := int64(level) + return strconv.Itoa(level), &numeric, nil + default: + return "", nil, fmt.Errorf("unknown config key: %s", key) + } +} + +func int64Pointer(value int64) *int64 { return &value } diff --git a/internal/engine/repository_configuration_test.go b/internal/engine/repository_configuration_test.go new file mode 100644 index 00000000..ce9ac973 --- /dev/null +++ b/internal/engine/repository_configuration_test.go @@ -0,0 +1,104 @@ +package engine_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/chunk" + "github.com/franchoy/coldkeep/internal/engine" +) + +func TestEngineConfigurationRoundTripsAndChangedSemantics(t *testing.T) { + dbconn := openSnapshotTestDB(t) + eng, err := engine.New(engine.Config{DB: dbconn}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + + for _, tc := range []struct { + key engine.ConfigurationKey + value string + want string + level int64 + }{ + {engine.ConfigurationDefaultChunker, string(chunk.VersionV1SimpleRolling), string(chunk.VersionV1SimpleRolling), 0}, + {engine.ConfigurationCompression, " zstd ", "zstd", 0}, + {engine.ConfigurationCompressionLevel, "5", "5", 5}, + } { + first, err := eng.SetConfiguration(context.Background(), engine.SetConfigurationRequest{Key: tc.key, Value: tc.value}) + if err != nil || !first.Changed || first.Value != tc.want { + t.Fatalf("SetConfiguration(%s): got (%+v, %v)", tc.key, first, err) + } + second, err := eng.SetConfiguration(context.Background(), engine.SetConfigurationRequest{Key: tc.key, Value: tc.value}) + if err != nil || second.Changed { + t.Fatalf("unchanged SetConfiguration(%s): got (%+v, %v)", tc.key, second, err) + } + got, err := eng.GetConfiguration(context.Background(), engine.GetConfigurationRequest{Key: tc.key}) + if err != nil || got.Value != tc.want { + t.Fatalf("GetConfiguration(%s): got (%+v, %v)", tc.key, got, err) + } + if tc.level != 0 && (got.IntegerValue == nil || *got.IntegerValue != tc.level) { + t.Fatalf("GetConfiguration(%s) integer projection: %+v", tc.key, got) + } + } +} + +func TestEngineConfigurationDefaultsAndValidation(t *testing.T) { + dbconn := openSnapshotTestDB(t) + if _, err := dbconn.Exec(`DELETE FROM repository_config`); err != nil { + t.Fatalf("delete repository configuration: %v", err) + } + eng, err := engine.New(engine.Config{DB: dbconn}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + for key, want := range map[engine.ConfigurationKey]string{ + engine.ConfigurationDefaultChunker: string(chunk.DefaultChunkerVersion), + engine.ConfigurationCompression: "none", + engine.ConfigurationCompressionLevel: "3", + } { + got, err := eng.GetConfiguration(context.Background(), engine.GetConfigurationRequest{Key: key}) + if err != nil || got.Value != want { + t.Fatalf("default GetConfiguration(%s): got (%+v, %v), want %q", key, got, err, want) + } + } + for _, req := range []engine.SetConfigurationRequest{ + {Key: "unknown", Value: "x"}, + {Key: engine.ConfigurationDefaultChunker, Value: "v9-future-cdc"}, + {Key: engine.ConfigurationCompression, Value: "xz"}, + {Key: engine.ConfigurationCompressionLevel, Value: "0"}, + {Key: engine.ConfigurationCompressionLevel, Value: "bad"}, + } { + if _, err := eng.SetConfiguration(context.Background(), req); !engine.IsCode(err, engine.ErrorInvalidArgument) { + t.Fatalf("invalid SetConfiguration(%+v): %v", req, err) + } + } +} + +func TestEngineConfigurationDeprecationAndCancellation(t *testing.T) { + dbconn := openSnapshotTestDB(t) + eng, err := engine.New(engine.Config{ + DB: dbconn, + ChunkerDeprecationPolicy: func(version chunk.Version) (bool, string) { + return version == chunk.VersionV2FastCDC, "scheduled removal" + }, + }) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + _, err = eng.SetConfiguration(context.Background(), engine.SetConfigurationRequest{ + Key: engine.ConfigurationDefaultChunker, Value: string(chunk.VersionV2FastCDC), + }) + if !engine.IsCode(err, engine.ErrorInvalidArgument) || !strings.Contains(err.Error(), "scheduled removal") { + t.Fatalf("deprecated chunker: %v", err) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + _, err = eng.GetConfiguration(cancelled, engine.GetConfigurationRequest{Key: engine.ConfigurationCompression}) + if !errors.Is(err, context.Canceled) || !engine.IsCode(err, engine.ErrorCancelled) { + t.Fatalf("cancelled GetConfiguration: %v", err) + } +} From 962a5bbad5152bd7eaf670d5ca27d8d78c46445e Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 14:32:27 +0200 Subject: [PATCH 14/27] Complete snapshot read-side engine ownership --- .../legacy_snapshot_read_seams_test.go | 228 +++++++++++ cmd/coldkeep/main.go | 358 ++++++++++-------- cmd/coldkeep/snapshot_engine_routing_test.go | 7 +- .../snapshot_read_engine_boundary_test.go | 138 +++++++ cmd/coldkeep/stored_path_engine_adapters.go | 4 + .../stored_path_engine_routing_test.go | 32 ++ docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...2-phase13-snapshot-read-side-completion.md | 60 +++ internal/engine/candidates.go | 74 +--- .../engine/contract_surface_clarity_test.go | 24 +- internal/engine/contracts_test.go | 4 +- internal/engine/default_engine.go | 76 +++- .../engine/default_engine_routing_helpers.go | 14 +- ...snapshot_selector_backend_contract_test.go | 17 +- 14 files changed, 792 insertions(+), 250 deletions(-) create mode 100644 cmd/coldkeep/legacy_snapshot_read_seams_test.go create mode 100644 cmd/coldkeep/snapshot_read_engine_boundary_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase13-snapshot-read-side-completion.md diff --git a/cmd/coldkeep/legacy_snapshot_read_seams_test.go b/cmd/coldkeep/legacy_snapshot_read_seams_test.go new file mode 100644 index 00000000..d6b51be9 --- /dev/null +++ b/cmd/coldkeep/legacy_snapshot_read_seams_test.go @@ -0,0 +1,228 @@ +package main + +import ( + "context" + "database/sql" + "regexp" + "time" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/snapshot" + "github.com/franchoy/coldkeep/internal/storage" +) + +// These seams exist only so pre-v1.13.12 CLI compatibility tests can keep +// supplying deterministic snapshot-domain fixtures. Production code has no +// direct snapshot read seam: it constructs and calls an Engine. +var listSnapshotsPhase = snapshot.ListSnapshots +var getSnapshotPhase = snapshot.GetSnapshot +var listSnapshotFilesPhase = snapshot.ListSnapshotFiles +var snapshotStatsPhase = snapshot.GetSnapshotStats +var diffSnapshotsPhase = snapshot.DiffSnapshots +var diffSnapshotSummaryPhase = snapshot.DiffSnapshotsSummarySQL + +var productionSnapshotReadCommandEngine = newSnapshotReadCommandEngine + +func init() { + newSnapshotReadCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return legacySnapshotReadTestEngine{db: sgctx.DB}, nil + } +} + +type legacySnapshotReadTestEngine struct { + engine.Engine + db *sql.DB +} + +func (e legacySnapshotReadTestEngine) SnapshotList(ctx context.Context, req engine.SnapshotListRequest) (engine.SnapshotListResult, error) { + filter := snapshot.SnapshotListFilter{Since: req.Since, Until: req.Until, Limit: req.Limit, Tree: req.Tree} + if req.Type != "" { + value := string(req.Type) + filter.Type = &value + } + if req.Label != "" { + value := req.Label + filter.Label = &value + } + items, err := listSnapshotsPhase(ctx, e.db, filter) + if err != nil { + return engine.SnapshotListResult{}, err + } + result := engine.SnapshotListResult{Snapshots: make([]engine.SnapshotMeta, len(items)), Count: len(items), TreeMode: req.Tree} + for i, item := range items { + result.Snapshots[i] = snapshotMetaFromLegacy(item) + } + if req.Tree { + result.Graph = legacySnapshotGraph(items) + } + return result, nil +} + +func (e legacySnapshotReadTestEngine) SnapshotShow(ctx context.Context, req engine.SnapshotShowRequest) (engine.SnapshotShowResult, error) { + item, err := getSnapshotPhase(ctx, e.db, req.SnapshotID) + if err != nil { + return engine.SnapshotShowResult{}, err + } + query, err := legacySnapshotQuery(req.Query) + if err != nil { + return engine.SnapshotShowResult{}, err + } + files, err := listSnapshotFilesPhase(ctx, e.db, req.SnapshotID, req.Query.Limit, query) + if err != nil { + return engine.SnapshotShowResult{}, err + } + stats, err := snapshotStatsPhase(ctx, e.db, req.SnapshotID) + if err != nil { + return engine.SnapshotShowResult{}, err + } + result := engine.SnapshotShowResult{ + Snapshot: snapshotMetaFromLegacy(*item), + Files: make([]engine.SnapshotFile, len(files)), + MatchedFileCount: len(files), + TotalFileCount: int(stats.SnapshotFileCount), + } + for i, file := range files { + result.Files[i] = engine.SnapshotFile{ + StoredPath: file.Path, LogicalFileID: file.LogicalFileID, + Size: nullableInt64FromLegacy(file.Size), Mode: nullableInt64FromLegacy(file.Mode), + ModTime: nullableTimeFromLegacy(file.MTime), + } + } + return result, nil +} + +func (e legacySnapshotReadTestEngine) SnapshotStats(ctx context.Context, req engine.SnapshotStatsRequest) (engine.SnapshotStatsResult, error) { + stats, err := snapshotStatsPhase(ctx, e.db, req.SnapshotID) + if err != nil { + return engine.SnapshotStatsResult{}, err + } + return engine.SnapshotStatsResult{ + SnapshotCount: int(stats.SnapshotCount), SnapshotFileCount: int(stats.SnapshotFileCount), + TotalSizeBytes: stats.TotalSizeBytes, HasReuse: stats.ParentSnapshotID.Valid, + Reused: int(stats.ReusedFileCount.Int64), New: int(stats.NewFileCount.Int64), + ReuseRatio: stats.ReuseRatioPct.Float64, LineageStatus: string(stats.LineageStatus), + ParentSnapshotID: stats.ParentSnapshotID.String, + }, nil +} + +func (e legacySnapshotReadTestEngine) SnapshotDiff(ctx context.Context, req engine.SnapshotDiffRequest) (engine.SnapshotDiffResult, error) { + if req.Summary && req.Filter == "" && isLegacySnapshotQueryEmpty(req.Query) { + summary, err := diffSnapshotSummaryPhase(ctx, e.db, req.BaseID, req.TargetID) + if err != nil { + return engine.SnapshotDiffResult{}, err + } + total := int(summary.Added + summary.Removed + summary.Modified) + return engine.SnapshotDiffResult{ + BaseID: req.BaseID, TargetID: req.TargetID, SummaryMode: true, + Summary: engine.SnapshotDiffSummary{Added: int(summary.Added), Removed: int(summary.Removed), Modified: int(summary.Modified)}, + MatchedEntryCount: total, TotalEntryCount: total, + }, nil + } + query, err := legacySnapshotQuery(req.Query) + if err != nil { + return engine.SnapshotDiffResult{}, err + } + raw, err := diffSnapshotsPhase(ctx, e.db, req.BaseID, req.TargetID, query) + if err != nil { + return engine.SnapshotDiffResult{}, err + } + result := engine.SnapshotDiffResult{BaseID: req.BaseID, TargetID: req.TargetID, SummaryMode: req.Summary, TotalEntryCount: len(raw.Entries)} + for _, item := range raw.Entries { + if req.Filter != "" && string(req.Filter) != string(item.Type) { + continue + } + result.MatchedEntryCount++ + switch item.Type { + case snapshot.DiffAdded: + result.Summary.Added++ + case snapshot.DiffRemoved: + result.Summary.Removed++ + case snapshot.DiffModified: + result.Summary.Modified++ + } + if !req.Summary { + result.Entries = append(result.Entries, engine.SnapshotDiffEntry{ + StoredPath: item.Path, Change: engine.SnapshotDiffChange(item.Type), + BaseLogicalID: nullableInt64FromLegacy(item.BaseLogicalID), + TargetLogicalID: nullableInt64FromLegacy(item.TargetLogicalID), + }) + } + } + return result, nil +} + +func snapshotMetaFromLegacy(item snapshot.Snapshot) engine.SnapshotMeta { + return engine.SnapshotMeta{ + ID: item.ID, Type: engine.SnapshotType(item.Type), Label: item.Label.String, + ParentID: item.ParentID.String, CreatedAt: item.CreatedAt, + } +} + +func legacySnapshotGraph(items []snapshot.Snapshot) *engine.SnapshotGraph { + ordered := snapshotSortAscending(items) + result := &engine.SnapshotGraph{Nodes: make([]engine.SnapshotGraphNode, len(ordered)), RootIDs: make([]string, 0)} + index := make(map[string]int, len(ordered)) + for i, item := range ordered { + index[item.ID] = i + result.Nodes[i] = engine.SnapshotGraphNode{Snapshot: snapshotMetaFromLegacy(item), ChildIDs: make([]string, 0)} + } + for i, item := range ordered { + if !item.ParentID.Valid { + result.Nodes[i].ParentState = engine.SnapshotParentNone + result.RootIDs = append(result.RootIDs, item.ID) + continue + } + parentIndex, ok := index[item.ParentID.String] + if !ok { + result.Nodes[i].ParentState = engine.SnapshotParentMissing + result.RootIDs = append(result.RootIDs, item.ID) + continue + } + result.Nodes[i].ParentState = engine.SnapshotParentPresent + result.Nodes[parentIndex].ChildIDs = append(result.Nodes[parentIndex].ChildIDs, item.ID) + } + return result +} + +func legacySnapshotQuery(query engine.SnapshotQuery) (*snapshot.SnapshotQuery, error) { + if isLegacySnapshotQueryEmpty(query) { + return nil, nil + } + result := &snapshot.SnapshotQuery{ + ExactPaths: make(map[string]struct{}, len(query.Paths)), Prefixes: append([]string(nil), query.Prefixes...), + Pattern: query.Pattern, MinSize: query.MinSize, MaxSize: query.MaxSize, + ModifiedAfter: query.ModifiedAfter, ModifiedBefore: query.ModifiedBefore, + } + for _, item := range query.Paths { + result.ExactPaths[item] = struct{}{} + } + if query.Regex != "" { + compiled, err := regexp.Compile(query.Regex) + if err != nil { + return nil, err + } + result.Regex = compiled + } + return result, nil +} + +func isLegacySnapshotQueryEmpty(query engine.SnapshotQuery) bool { + return len(query.Paths) == 0 && len(query.Prefixes) == 0 && query.Pattern == "" && query.Regex == "" && + query.MinSize == nil && query.MaxSize == nil && query.ModifiedAfter == nil && query.ModifiedBefore == nil && query.Limit == 0 +} + +func nullableInt64FromLegacy(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + result := value.Int64 + return &result +} + +func nullableTimeFromLegacy(value sql.NullTime) *time.Time { + if !value.Valid { + return nil + } + result := value.Time + return &result +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index c0144b5e..689d04dd 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -367,111 +367,12 @@ var createSnapshotPhase = snapshot.CreateSnapshotWithOptions // through Engine.SnapshotRestore. var restoreSnapshotPhase = snapshot.RestoreSnapshot var currentWorkingDirectoryPhase = os.Getwd -var listSnapshotsPhase = func(ctx context.Context, db *sql.DB, filter snapshot.SnapshotListFilter) ([]snapshot.Snapshot, error) { - eng, err := engine.New(engine.Config{DB: db}) - if err != nil { - return nil, err - } - req := engine.SnapshotListRequest{ - Since: filter.Since, - Until: filter.Until, - Limit: filter.Limit, - Tree: filter.Tree, - } - if filter.Type != nil { - req.Type = engine.SnapshotType(*filter.Type) - } - if filter.Label != nil { - req.Label = *filter.Label - } - result, err := eng.SnapshotList(ctx, req) - if err != nil { - return nil, err - } - items := make([]snapshot.Snapshot, len(result.Snapshots)) - for i, m := range result.Snapshots { - items[i] = snapshotMetaToSnapshot(m) - } - return items, nil -} -var getSnapshotPhase = func(ctx context.Context, db *sql.DB, id string) (*snapshot.Snapshot, error) { - // Engine.SnapshotShow is active, but the CLI workflow may still combine - // engine metadata with direct snapshot-domain listing, counting, or - // rendering. Active method presence does not prove complete read-side - // workflow ownership; early v2.0 owns the remaining ownership decision. - eng, err := engine.New(engine.Config{DB: db}) - if err != nil { - return nil, err - } - result, err := eng.SnapshotShow(ctx, engine.SnapshotShowRequest{SnapshotID: id}) - if err != nil { - return nil, err - } - s := snapshotMetaToSnapshot(result.Snapshot) - return &s, nil -} - -// Engine.SnapshotShow is active, while snapshot show file listing remains -// direct snapshot-domain work in this mixed CLI workflow. Active method -// presence does not prove complete read-side workflow ownership; early v2.0 -// owns the remaining ownership decision. -var listSnapshotFilesPhase = snapshot.ListSnapshotFiles -var snapshotStatsPhase = func(ctx context.Context, db *sql.DB, id string) (*snapshot.SnapshotStats, error) { - // Engine.SnapshotStats is active, but snapshot show and stats workflows may - // still use direct snapshot-domain helpers alongside engine-backed seams. - // Active method presence does not prove complete read-side workflow - // ownership; early v2.0 owns the remaining ownership decision. - eng, err := engine.New(engine.Config{DB: db}) - if err != nil { - return nil, err - } - result, err := eng.SnapshotStats(ctx, engine.SnapshotStatsRequest{SnapshotID: id}) - if err != nil { - return nil, err - } - stats := &snapshot.SnapshotStats{ - SnapshotCount: int64(result.SnapshotCount), - SnapshotFileCount: int64(result.SnapshotFileCount), - TotalSizeBytes: result.TotalSizeBytes, - LineageStatus: snapshot.SnapshotLineageStatus(result.LineageStatus), - } - if result.HasReuse { - stats.ParentSnapshotID = sql.NullString{Valid: true, String: result.ParentSnapshotID} - stats.ReusedFileCount = sql.NullInt64{Valid: true, Int64: int64(result.Reused)} - stats.NewFileCount = sql.NullInt64{Valid: true, Int64: int64(result.New)} - stats.ReuseRatioPct = sql.NullFloat64{Valid: true, Float64: result.ReuseRatio} - } - return stats, nil -} // Compatibility-only direct snapshot-domain delete seams remain for lower- // level tests and non-CLI callers. Production CLI snapshot delete routes // through Engine.SnapshotDelete. var deleteSnapshotPhase = snapshot.DeleteSnapshot var snapshotDeleteLineagePreviewPhase = loadSnapshotDeleteLineagePreview -var diffSnapshotsPhase = func(ctx context.Context, db *sql.DB, baseID, targetID string, query *snapshot.SnapshotQuery) (*snapshot.SnapshotDiffResult, error) { - // The CLI accepts repeated path and prefix selectors. Keep that full - // snapshot-domain query shape instead of narrowing it through the current - // single-path engine seam. - return snapshot.DiffSnapshots(ctx, db, baseID, targetID, query) -} -var diffSnapshotSummaryPhase = func(ctx context.Context, db *sql.DB, baseID, targetID string) (*snapshot.SnapshotDiffSummary, error) { - eng, err := engine.New(engine.Config{DB: db}) - if err != nil { - return nil, err - } - result, err := eng.SnapshotDiff(ctx, engine.SnapshotDiffRequest{ - BaseID: baseID, TargetID: targetID, Summary: true, - }) - if err != nil { - return nil, err - } - return &snapshot.SnapshotDiffSummary{ - Added: int64(result.Summary.Added), - Removed: int64(result.Summary.Removed), - Modified: int64(result.Summary.Modified), - }, nil -} // snapshotMetaToSnapshot maps an engine.SnapshotMeta to a snapshot.Snapshot for // CLI renderers that expect the snapshot package's type with sql.NullString fields. @@ -5566,6 +5467,70 @@ func renderSnapshotTreeLines(items []snapshot.Snapshot) []string { return lines } +func renderEngineSnapshotTreeLines(graph *engine.SnapshotGraph) []string { + if graph == nil || len(graph.Nodes) == 0 { + return nil + } + + nodes := make(map[string]engine.SnapshotGraphNode, len(graph.Nodes)) + for _, node := range graph.Nodes { + nodes[node.Snapshot.ID] = node + } + lines := make([]string, 0, len(graph.Nodes)) + visited := make(map[string]struct{}, len(graph.Nodes)) + var walk func(string, string, bool, bool) + walk = func(id, prefix string, isLast, hasParent bool) { + if _, seen := visited[id]; seen { + return + } + node, ok := nodes[id] + if !ok { + return + } + visited[id] = struct{}{} + + linePrefix := prefix + if hasParent { + if isLast { + linePrefix += "└── " + } else { + linePrefix += "├── " + } + } + lines = append(lines, linePrefix+id) + + nextPrefix := prefix + if hasParent { + if isLast { + nextPrefix += " " + } else { + nextPrefix += "│ " + } + } + for i, childID := range node.ChildIDs { + walk(childID, nextPrefix, i == len(node.ChildIDs)-1, true) + } + } + emitRoot := func(id string) { + if _, seen := visited[id]; seen { + return + } + if len(lines) > 0 { + lines = append(lines, "") + } + walk(id, "", true, false) + } + for _, rootID := range graph.RootIDs { + emitRoot(rootID) + } + // Defensive completion keeps rendering finite if a custom Engine returns a + // malformed root list; relationship validation remains engine-owned. + for _, node := range graph.Nodes { + emitRoot(node.Snapshot.ID) + } + return lines +} + func snapshotFilesJSON(items []snapshot.SnapshotFileEntry) []map[string]any { result := make([]map[string]any, 0, len(items)) for _, item := range items { @@ -5580,6 +5545,77 @@ func snapshotFilesJSON(items []snapshot.SnapshotFileEntry) []map[string]any { return result } +func snapshotQueryToEngine(query *snapshot.SnapshotQuery, limit int) engine.SnapshotQuery { + result := engine.SnapshotQuery{Limit: limit} + if query == nil { + return result + } + result.Paths = make([]string, 0, len(query.ExactPaths)) + for exactPath := range query.ExactPaths { + result.Paths = append(result.Paths, exactPath) + } + sort.Strings(result.Paths) + result.Prefixes = append([]string(nil), query.Prefixes...) + result.Pattern = query.Pattern + if query.Regex != nil { + result.Regex = query.Regex.String() + } + result.MinSize = query.MinSize + result.MaxSize = query.MaxSize + result.ModifiedAfter = query.ModifiedAfter + result.ModifiedBefore = query.ModifiedBefore + return result +} + +func snapshotFileFromEngine(file engine.SnapshotFile) snapshot.SnapshotFileEntry { + result := snapshot.SnapshotFileEntry{Path: file.StoredPath, LogicalFileID: file.LogicalFileID} + if file.Size != nil { + result.Size = sql.NullInt64{Int64: *file.Size, Valid: true} + } + if file.Mode != nil { + result.Mode = sql.NullInt64{Int64: *file.Mode, Valid: true} + } + if file.ModTime != nil { + result.MTime = sql.NullTime{Time: *file.ModTime, Valid: true} + } + return result +} + +func snapshotStatsFromEngine(result engine.SnapshotStatsResult) snapshot.SnapshotStats { + stats := snapshot.SnapshotStats{ + SnapshotCount: int64(result.SnapshotCount), SnapshotFileCount: int64(result.SnapshotFileCount), + TotalSizeBytes: result.TotalSizeBytes, LineageStatus: snapshot.SnapshotLineageStatus(result.LineageStatus), + } + if result.HasReuse { + stats.ParentSnapshotID = sql.NullString{String: result.ParentSnapshotID, Valid: true} + stats.ReusedFileCount = sql.NullInt64{Int64: int64(result.Reused), Valid: true} + stats.NewFileCount = sql.NullInt64{Int64: int64(result.New), Valid: true} + stats.ReuseRatioPct = sql.NullFloat64{Float64: result.ReuseRatio, Valid: true} + } + return stats +} + +func snapshotDiffFromEngine(result engine.SnapshotDiffResult) snapshot.SnapshotDiffResult { + entries := make([]snapshot.SnapshotDiffEntry, len(result.Entries)) + for i, entry := range result.Entries { + entries[i] = snapshot.SnapshotDiffEntry{ + Path: entry.StoredPath, Type: snapshot.DiffType(entry.Change), + BaseLogicalID: nullableSQLInt64(entry.BaseLogicalID), TargetLogicalID: nullableSQLInt64(entry.TargetLogicalID), + } + } + return snapshot.SnapshotDiffResult{ + BaseSnapshotID: result.BaseID, TargetSnapshotID: result.TargetID, Entries: entries, + Summary: snapshot.SnapshotDiffSummary{Added: int64(result.Summary.Added), Removed: int64(result.Summary.Removed), Modified: int64(result.Summary.Modified)}, + } +} + +func nullableSQLInt64(value *int64) sql.NullInt64 { + if value == nil { + return sql.NullInt64{} + } + return sql.NullInt64{Int64: *value, Valid: true} +} + func runSnapshotListCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { startedAt := time.Now() @@ -5644,10 +5680,34 @@ func runSnapshotListCommand(parsed parsedCommandLine, outputMode cliOutputMode) ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() - items, err := listSnapshotsPhase(ctx, sgctx.DB, filter) + eng, err := newSnapshotReadCommandEngine(sgctx) if err != nil { return err } + req := engine.SnapshotListRequest{ + Since: filter.Since, Until: filter.Until, Limit: filter.Limit, Tree: treeMode, + } + if filter.Type != nil { + req.Type = engine.SnapshotType(*filter.Type) + } + if filter.Label != nil { + req.Label = *filter.Label + } + result, err := eng.SnapshotList(ctx, req) + if err != nil { + return err + } + items := make([]snapshot.Snapshot, len(result.Snapshots)) + for i, item := range result.Snapshots { + items[i] = snapshotMetaToSnapshot(item) + } + var treeLines []string + if treeMode { + if result.Graph == nil { + return fmt.Errorf("snapshot list engine result missing requested graph") + } + treeLines = renderEngineSnapshotTreeLines(result.Graph) + } if outputMode == outputModeJSON { jsonItems := make([]map[string]any, 0, len(items)) @@ -5662,7 +5722,7 @@ func runSnapshotListCommand(parsed parsedCommandLine, outputMode cliOutputMode) } if treeMode { data["tree_mode"] = true - data["tree_lines"] = renderSnapshotTreeLines(items) + data["tree_lines"] = treeLines } payload := map[string]any{ "status": "ok", @@ -5675,8 +5735,7 @@ func runSnapshotListCommand(parsed parsedCommandLine, outputMode cliOutputMode) } if treeMode { - lines := renderSnapshotTreeLines(items) - return clirender.RenderSnapshotListHuman(os.Stdout, items, true, lines, time.Since(startedAt).Milliseconds(), doctorOperationalHint) + return clirender.RenderSnapshotListHuman(os.Stdout, items, true, treeLines, time.Since(startedAt).Milliseconds(), doctorOperationalHint) } return clirender.RenderSnapshotListHuman(os.Stdout, items, false, nil, time.Since(startedAt).Milliseconds(), doctorOperationalHint) @@ -5718,19 +5777,23 @@ func runSnapshotShowCommand(parsed parsedCommandLine, outputMode cliOutputMode) ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() - item, err := getSnapshotPhase(ctx, sgctx.DB, snapshotID) + eng, err := newSnapshotReadCommandEngine(sgctx) if err != nil { return err } - files, err := listSnapshotFilesPhase(ctx, sgctx.DB, snapshotID, limit, query) + result, err := eng.SnapshotShow(ctx, engine.SnapshotShowRequest{ + SnapshotID: snapshotID, + Query: snapshotQueryToEngine(query, limit), + }) if err != nil { return err } - stats, err := snapshotStatsPhase(ctx, sgctx.DB, snapshotID) - if err != nil { - return err + item := snapshotMetaToSnapshot(result.Snapshot) + files := make([]snapshot.SnapshotFileEntry, len(result.Files)) + for i, file := range result.Files { + files[i] = snapshotFileFromEngine(file) } - matchedFileCount := len(files) + matchedFileCount := result.MatchedFileCount if outputMode == outputModeJSON { payload := map[string]any{ @@ -5738,10 +5801,10 @@ func runSnapshotShowCommand(parsed parsedCommandLine, outputMode cliOutputMode) "command": "snapshot", "data": map[string]any{ "action": "show", - "snapshot": snapshotSummaryJSON(*item), + "snapshot": snapshotSummaryJSON(item), "file_count": matchedFileCount, "matched_file_count": matchedFileCount, - "total_snapshot_file_count": stats.SnapshotFileCount, + "total_snapshot_file_count": result.TotalFileCount, "files": snapshotFilesJSON(files), "duration_ms": time.Since(startedAt).Milliseconds(), }, @@ -5753,10 +5816,10 @@ func runSnapshotShowCommand(parsed parsedCommandLine, outputMode cliOutputMode) return clirender.RenderSnapshotShowHuman( os.Stdout, - *item, + item, files, matchedFileCount, - stats.SnapshotFileCount, + int64(result.TotalFileCount), time.Since(startedAt).Milliseconds(), doctorOperationalHint, ) @@ -5789,10 +5852,15 @@ func runSnapshotStatsCommand(parsed parsedCommandLine, outputMode cliOutputMode) ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() - stats, err := snapshotStatsPhase(ctx, sgctx.DB, snapshotID) + eng, err := newSnapshotReadCommandEngine(sgctx) + if err != nil { + return err + } + result, err := eng.SnapshotStats(ctx, engine.SnapshotStatsRequest{SnapshotID: snapshotID}) if err != nil { return err } + stats := snapshotStatsFromEngine(result) if outputMode == outputModeJSON { data := map[string]any{ @@ -6142,60 +6210,22 @@ func runSnapshotDiffCommand(parsed parsedCommandLine, outputMode cliOutputMode) ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() - useSummaryFastPath := summaryMode && filterType == "" && query == nil - if useSummaryFastPath { - summary, err := diffSnapshotSummaryPhase(ctx, sgctx.DB, baseID, targetID) - if err != nil { - return err - } - totalEntryCount := int(summary.Added + summary.Removed + summary.Modified) - - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "snapshot diff", - "data": map[string]any{ - "base": baseID, - "target": targetID, - "entry_count": totalEntryCount, - "matched_entry_count": totalEntryCount, - "total_diff_entry_count": totalEntryCount, - "summary": summary, - "summary_mode": true, - "duration_ms": time.Since(startedAt).Milliseconds(), - }, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil - } - - return clirender.RenderSnapshotDiffSummaryHuman(os.Stdout, baseID, targetID, *summary) - } - - result, err := diffSnapshotsPhase(ctx, sgctx.DB, baseID, targetID, query) + eng, err := newSnapshotReadCommandEngine(sgctx) if err != nil { return err } - - entries := make([]snapshot.SnapshotDiffEntry, 0, len(result.Entries)) - summary := snapshot.SnapshotDiffSummary{} - for _, entry := range result.Entries { - if filterType != "" && entry.Type != snapshot.DiffType(filterType) { - continue - } - entries = append(entries, entry) - switch entry.Type { - case snapshot.DiffAdded: - summary.Added++ - case snapshot.DiffRemoved: - summary.Removed++ - case snapshot.DiffModified: - summary.Modified++ - } + engineResult, err := eng.SnapshotDiff(ctx, engine.SnapshotDiffRequest{ + BaseID: baseID, TargetID: targetID, Summary: summaryMode, + Filter: engine.SnapshotDiffFilter(filterType), Query: snapshotQueryToEngine(query, 0), + }) + if err != nil { + return err } - totalEntryCount := len(result.Entries) - matchedEntryCount := len(entries) + result := snapshotDiffFromEngine(engineResult) + entries := result.Entries + summary := result.Summary + totalEntryCount := engineResult.TotalEntryCount + matchedEntryCount := engineResult.MatchedEntryCount if outputMode == outputModeJSON { jsonEntries := make([]map[string]any, 0, len(entries)) diff --git a/cmd/coldkeep/snapshot_engine_routing_test.go b/cmd/coldkeep/snapshot_engine_routing_test.go index 95cf6367..0687f4b4 100644 --- a/cmd/coldkeep/snapshot_engine_routing_test.go +++ b/cmd/coldkeep/snapshot_engine_routing_test.go @@ -39,10 +39,15 @@ func openSnapshotRoutingDB(t *testing.T) *sql.DB { func injectSnapshotRoutingDB(t *testing.T, dbconn *sql.DB) { t.Helper() orig := loadDefaultStorageContextPhase - t.Cleanup(func() { loadDefaultStorageContextPhase = orig }) + origEngine := newSnapshotReadCommandEngine + t.Cleanup(func() { + loadDefaultStorageContextPhase = orig + newSnapshotReadCommandEngine = origEngine + }) loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { return storage.StorageContext{DB: dbconn}, nil } + newSnapshotReadCommandEngine = productionSnapshotReadCommandEngine } // insertRoutingSnapshot inserts a minimal snapshot row for routing tests. diff --git a/cmd/coldkeep/snapshot_read_engine_boundary_test.go b/cmd/coldkeep/snapshot_read_engine_boundary_test.go new file mode 100644 index 00000000..0ca4d5fb --- /dev/null +++ b/cmd/coldkeep/snapshot_read_engine_boundary_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" +) + +func installSnapshotReadEngineStub(t *testing.T, stub engine.Engine) { + t.Helper() + dbconn := openSnapshotRoutingDB(t) + originalLoad := loadDefaultStorageContextPhase + originalFactory := newSnapshotReadCommandEngine + t.Cleanup(func() { + loadDefaultStorageContextPhase = originalLoad + newSnapshotReadCommandEngine = originalFactory + }) + loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { + return storage.StorageContext{DB: dbconn}, nil + } + newSnapshotReadCommandEngine = func(storage.StorageContext) (engine.Engine, error) { return stub, nil } +} + +func TestSnapshotShowUsesOneTypedEngineOperationWithRepeatedSelectors(t *testing.T) { + calls := 0 + installSnapshotReadEngineStub(t, stubCommandEngine{ + snapshotShowFunc: func(_ context.Context, req engine.SnapshotShowRequest) (engine.SnapshotShowResult, error) { + calls++ + if req.SnapshotID != "snap-read" { + t.Fatalf("snapshot id = %q", req.SnapshotID) + } + if !reflect.DeepEqual(req.Query.Paths, []string{"a.txt", "b.txt"}) { + t.Fatalf("paths = %#v", req.Query.Paths) + } + if !reflect.DeepEqual(req.Query.Prefixes, []string{"docs/", "src/"}) { + t.Fatalf("prefixes = %#v", req.Query.Prefixes) + } + if req.Query.Pattern != "*.txt" || req.Query.Regex != `^(a|b)` || req.Query.Limit != 7 { + t.Fatalf("query = %+v", req.Query) + } + size := int64(9007199254740993) + return engine.SnapshotShowResult{ + Snapshot: engine.SnapshotMeta{ID: req.SnapshotID, Type: engine.SnapshotTypeFull, FileCount: 1}, + Files: []engine.SnapshotFile{{StoredPath: "a.txt", LogicalFileID: 11, Size: &size}}, + MatchedFileCount: 1, TotalFileCount: 2, + }, nil + }, + }) + + output := captureStdout(t, func() { + err := runSnapshotCommand(parsedCommandLine{ + method: "snapshot", positionals: []string{"show", "snap-read"}, + flags: map[string][]string{ + "path": {"b.txt", "a.txt"}, "prefix": {"docs/", "src/"}, + "pattern": {"*.txt"}, "regex": {`^(a|b)`}, "limit": {"7"}, "output": {"json"}, + }, + }, outputModeJSON) + if err != nil { + t.Fatalf("snapshot show: %v", err) + } + }) + if calls != 1 { + t.Fatalf("SnapshotShow calls = %d, want 1", calls) + } + if !strings.Contains(output, `"size":9007199254740993`) || !strings.Contains(output, `"total_snapshot_file_count":2`) { + t.Fatalf("compatibility JSON = %s", output) + } +} + +func TestSnapshotListStatsAndDiffEachUseOneEngineOperation(t *testing.T) { + now := time.Date(2026, 8, 19, 12, 0, 0, 0, time.UTC) + listCalls, statsCalls, diffCalls := 0, 0, 0 + installSnapshotReadEngineStub(t, stubCommandEngine{ + snapshotListFunc: func(_ context.Context, req engine.SnapshotListRequest) (engine.SnapshotListResult, error) { + listCalls++ + if req.Type != engine.SnapshotTypePartial || req.Label != "daily" || !req.Tree || req.Limit != 4 { + t.Fatalf("list request = %+v", req) + } + meta := engine.SnapshotMeta{ID: "s1", Type: engine.SnapshotTypePartial, CreatedAt: now} + return engine.SnapshotListResult{ + Snapshots: []engine.SnapshotMeta{meta}, Count: 1, TreeMode: true, + Graph: &engine.SnapshotGraph{Nodes: []engine.SnapshotGraphNode{{Snapshot: meta, ParentState: engine.SnapshotParentNone}}, RootIDs: []string{"s1"}}, + }, nil + }, + snapshotStatsFunc: func(_ context.Context, req engine.SnapshotStatsRequest) (engine.SnapshotStatsResult, error) { + statsCalls++ + if req.SnapshotID != "s1" { + t.Fatalf("stats request = %+v", req) + } + return engine.SnapshotStatsResult{SnapshotCount: 1, SnapshotFileCount: 2, TotalSizeBytes: 3}, nil + }, + snapshotDiffFunc: func(_ context.Context, req engine.SnapshotDiffRequest) (engine.SnapshotDiffResult, error) { + diffCalls++ + if req.BaseID != "s1" || req.TargetID != "s2" || !req.Summary || req.Filter != engine.SnapshotDiffAdded { + t.Fatalf("diff request = %+v", req) + } + if !reflect.DeepEqual(req.Query.Paths, []string{"a", "b"}) || !reflect.DeepEqual(req.Query.Prefixes, []string{"p/", "q/"}) { + t.Fatalf("diff query = %+v", req.Query) + } + return engine.SnapshotDiffResult{BaseID: "s1", TargetID: "s2", SummaryMode: true, Summary: engine.SnapshotDiffSummary{Added: 1}, MatchedEntryCount: 1, TotalEntryCount: 3}, nil + }, + }) + + commands := []parsedCommandLine{ + {method: "snapshot", positionals: []string{"list"}, flags: map[string][]string{"type": {"partial"}, "label": {"daily"}, "limit": {"4"}, "tree": {""}, "output": {"json"}}}, + {method: "snapshot", positionals: []string{"stats", "s1"}, flags: map[string][]string{"output": {"json"}}}, + {method: "snapshot", positionals: []string{"diff", "s1", "s2"}, flags: map[string][]string{"summary": {""}, "filter": {"added"}, "path": {"b", "a"}, "prefix": {"p/", "q/"}, "output": {"json"}}}, + } + for _, command := range commands { + captureStdout(t, func() { + if err := runSnapshotCommand(command, outputModeJSON); err != nil { + t.Fatalf("%v: %v", command.positionals, err) + } + }) + } + if listCalls != 1 || statsCalls != 1 || diffCalls != 1 { + t.Fatalf("calls list=%d stats=%d diff=%d", listCalls, statsCalls, diffCalls) + } +} + +func TestSnapshotReadEngineErrorPropagatesWithoutFallback(t *testing.T) { + want := errors.New("snapshot engine failure") + installSnapshotReadEngineStub(t, stubCommandEngine{ + snapshotShowFunc: func(context.Context, engine.SnapshotShowRequest) (engine.SnapshotShowResult, error) { + return engine.SnapshotShowResult{}, want + }, + }) + err := runSnapshotCommand(parsedCommandLine{method: "snapshot", positionals: []string{"show", "s1"}, flags: map[string][]string{}}, outputModeText) + if !errors.Is(err, want) { + t.Fatalf("error = %v, want %v", err, want) + } +} diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index b9a4b931..103368a4 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -40,6 +40,10 @@ var newConfigurationCommandEngine = func(sgctx storage.StorageContext) (engine.E return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) } +var newSnapshotReadCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 2f0e91c1..8d8e52ab 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -20,6 +20,10 @@ type stubCommandEngine struct { snapshotCreateFunc func(context.Context, engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) snapshotDeleteFunc func(context.Context, engine.SnapshotDeleteRequest) (engine.SnapshotDeleteResult, error) snapshotRestoreFunc func(context.Context, engine.SnapshotRestoreRequest) (engine.SnapshotRestoreResult, error) + snapshotListFunc func(context.Context, engine.SnapshotListRequest) (engine.SnapshotListResult, error) + snapshotShowFunc func(context.Context, engine.SnapshotShowRequest) (engine.SnapshotShowResult, error) + snapshotStatsFunc func(context.Context, engine.SnapshotStatsRequest) (engine.SnapshotStatsResult, error) + snapshotDiffFunc func(context.Context, engine.SnapshotDiffRequest) (engine.SnapshotDiffResult, error) restoreStoredPathFunc func(context.Context, engine.RestoreStoredPathRequest) (engine.RestoreStoredPathResult, error) removeStoredPathsFunc func(context.Context, engine.RemoveStoredPathsRequest) (engine.RemoveStoredPathsResult, error) storeFolderFunc func(context.Context, engine.StoreFolderRequest) (engine.StoreFolderResult, error) @@ -50,6 +54,34 @@ func (s stubCommandEngine) SnapshotRestore(ctx context.Context, req engine.Snaps return engine.SnapshotRestoreResult{}, errors.New("unexpected SnapshotRestore call") } +func (s stubCommandEngine) SnapshotList(ctx context.Context, req engine.SnapshotListRequest) (engine.SnapshotListResult, error) { + if s.snapshotListFunc != nil { + return s.snapshotListFunc(ctx, req) + } + return engine.SnapshotListResult{}, errors.New("unexpected SnapshotList call") +} + +func (s stubCommandEngine) SnapshotShow(ctx context.Context, req engine.SnapshotShowRequest) (engine.SnapshotShowResult, error) { + if s.snapshotShowFunc != nil { + return s.snapshotShowFunc(ctx, req) + } + return engine.SnapshotShowResult{}, errors.New("unexpected SnapshotShow call") +} + +func (s stubCommandEngine) SnapshotStats(ctx context.Context, req engine.SnapshotStatsRequest) (engine.SnapshotStatsResult, error) { + if s.snapshotStatsFunc != nil { + return s.snapshotStatsFunc(ctx, req) + } + return engine.SnapshotStatsResult{}, errors.New("unexpected SnapshotStats call") +} + +func (s stubCommandEngine) SnapshotDiff(ctx context.Context, req engine.SnapshotDiffRequest) (engine.SnapshotDiffResult, error) { + if s.snapshotDiffFunc != nil { + return s.snapshotDiffFunc(ctx, req) + } + return engine.SnapshotDiffResult{}, errors.New("unexpected SnapshotDiff call") +} + func (s stubCommandEngine) RestoreStoredPath(ctx context.Context, req engine.RestoreStoredPathRequest) (engine.RestoreStoredPathResult, error) { if s.restoreStoredPathFunc != nil { return s.restoreStoredPathFunc(ctx, req) diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 341c84dd..35d89355 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -87,11 +87,13 @@ ## Phase 13 — Snapshot Read-Side Completion -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 13 snapshot read-side completion](v1.13.12-phase13-snapshot-read-side-completion.md). ## Phase 14 — Inspect, Stats, and Verify Routing -**Status:** Not started +**Status:** Next ## Phase 15 — Repair Activation diff --git a/docs/release/v1.13/v1.13.12-phase13-snapshot-read-side-completion.md b/docs/release/v1.13/v1.13.12-phase13-snapshot-read-side-completion.md new file mode 100644 index 00000000..029a1e15 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase13-snapshot-read-side-completion.md @@ -0,0 +1,60 @@ +# Coldkeep v1.13.12 Phase 13 — Snapshot Read-Side Completion + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `13 — Snapshot Read-Side Completion` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Engine ownership + +Snapshot list, tree, show, stats, detailed diff, filtered diff, and summary diff +now each execute as one typed engine operation. The production CLI no longer +has direct snapshot metadata, file-list, statistics, detailed-diff, or +summary-diff seams. + +`SnapshotQuery` preserves repeated exact paths and prefixes. Pattern, regular +expression, size, time, and limit selectors cross the engine boundary in typed, +presentation-neutral fields. Snapshot files retain nullable size, mode, and +modification time; diff entries retain nullable base and target logical IDs. + +Tree rendering consumes the engine's selected graph projection. The engine +owns relationship validation, deterministic node/child/root ordering, +historical missing-parent classification, and effective roots when a selected +child's existing parent is filtered out. The CLI owns only tree glyphs and +success-envelope rendering. + +## Compatibility + +The established text and JSON envelopes, filtered and total counts, summary +mode, ordering, exact integer tokens, validation messages, and exit +classification remain unchanged. Test-only adapters preserve older fixture +injection points without restoring a production bypass. + +No schema, migration, repository format, stored bytes, snapshot metadata, +default backend, lock, or coordination behavior changed. + +## Validation + +- Direct CLI engine-boundary and compatibility suites: pass. +- Direct engine selector, ordering, cancellation, and error suites: pass. +- Shared SQLite/PostgreSQL snapshot selector contracts: pass. +- Focused CLI, engine, and snapshot race suites: pass. +- Full `go test -count=1 ./...`: pass. +- Validation root: `/tmp/coldkeep-v11312-phase13.4rNzyu/`, containing isolated + caches, temporary files, storage roots, and SQLite databases. + +PostgreSQL validation created and dropped exactly: + +- `coldkeep_bt_testenginesnapshotselectorsacrossbackends__2c9487_1` +- `coldkeep_bt_testenginesnapshotselectorerrorsacrossback_2c9487_2` +- `coldkeep_bt_testenginesnapshotselectorsacrossbackends__2e1d4a_1` +- `coldkeep_bt_testenginesnapshotselectorerrorsacrossback_2e1d4a_2` + +A read-only maintenance query confirmed both exact names are absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 14 is authorized to route inspect and finish stats/verify CLI ownership +without changing their public output, diagnostics, trace, or verification +semantics. diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index 0258fd01..a4f76658 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -4,12 +4,10 @@ import "time" // Engine operation contracts. // -// This historically named file contains both active Engine request/result -// contracts and the candidate-only corrective Repair and Recover contracts. -// Active method ownership is defined by the Engine interface, not by this file -// name. Some active read-side contracts remain provisional and may still -// participate in mixed CLI/domain workflows; active does not imply a frozen -// daemon/API-ready contract. +// This historically named file contains active Engine request/result contracts +// plus corrective Repair and Recover contracts that later v1.13.12 phases +// activate. Active method ownership is defined by the Engine interface, not by +// this file name. // // Contract rules (see docs/release/v1.12/engine-baseline.md): // - Requests represent operation intent, not CLI syntax. @@ -80,18 +78,11 @@ type BatchSummary struct { // snapshot show and diff. All fields are optional; zero values mean // "no filter on this dimension". Size and time fields use pointers so that a // zero value can be distinguished from "unset". -// -// Only one exact Path and one Prefix can cross the current read-side seam, -// even though CLI parsing may accept richer repeated path/prefix inputs before -// narrowing. Broader read-side shape and ownership decisions are deferred to -// early v2.0. type SnapshotQuery struct { - // Path matches an exact stored path. - // Only one exact path is preserved at the current engine seam. - Path string - // Prefix matches stored paths by prefix. - // Only one prefix is preserved at the current engine seam. - Prefix string + // Paths match exact normalized stored paths. + Paths []string + // Prefixes match normalized stored paths by directory prefix. + Prefixes []string // Pattern is a glob-style match against stored paths. Pattern string // Regex is a regular-expression match against stored paths. @@ -561,25 +552,19 @@ type SnapshotListResult struct { type SnapshotFile struct { StoredPath string LogicalFileID int64 - Size int64 - Mode uint32 - ModTime time.Time + Size *int64 + Mode *int64 + ModTime *time.Time } -// SnapshotShowRequest is the active but provisional request contract for -// Engine.SnapshotShow. +// SnapshotShowRequest is the active request contract for Engine.SnapshotShow. type SnapshotShowRequest struct { SnapshotID string // Query filters which files are returned. Query SnapshotQuery } -// SnapshotShowResult is the active but provisional result contract for -// Engine.SnapshotShow. -// -// This coherent result shape does not prove fully unified engine ownership. -// Metadata, listing, and counts may still come from mixed seams; read-side -// ownership and shape decisions are deferred to early v2.0. +// SnapshotShowResult is the active result contract for Engine.SnapshotShow. type SnapshotShowResult struct { Snapshot SnapshotMeta Files []SnapshotFile @@ -589,20 +574,15 @@ type SnapshotShowResult struct { TotalFileCount int } -// SnapshotStatsRequest is the active but provisional request contract for -// Engine.SnapshotStats. +// SnapshotStatsRequest is the active request contract for Engine.SnapshotStats. // // SnapshotID is optional; empty means aggregate stats across all snapshots. type SnapshotStatsRequest struct { SnapshotID string } -// SnapshotStatsResult is the active but provisional result contract for -// Engine.SnapshotStats. -// +// SnapshotStatsResult is the active result contract for Engine.SnapshotStats. // Reuse fields are populated only for a specific snapshot that has a parent. -// Aggregate, metadata, and rendering workflows may still use mixed seams; -// read-side ownership and shape decisions are deferred to early v2.0. type SnapshotStatsResult struct { SnapshotCount int SnapshotFileCount int @@ -646,17 +626,13 @@ const ( // SnapshotDiffEntry is a renderer-neutral diff entry. type SnapshotDiffEntry struct { - StoredPath string - Change SnapshotDiffChange + StoredPath string + Change SnapshotDiffChange + BaseLogicalID *int64 + TargetLogicalID *int64 } -// SnapshotDiffRequest is the active but provisional request contract for -// Engine.SnapshotDiff. -// -// Summary fast-path behavior and query/filter semantics remain provisional. -// The CLI can parse richer repeated path/prefix inputs than the current engine -// seam preserves; read-side ownership and shape decisions are deferred to -// early v2.0. +// SnapshotDiffRequest is the active request contract for Engine.SnapshotDiff. type SnapshotDiffRequest struct { BaseID string TargetID string @@ -665,8 +641,6 @@ type SnapshotDiffRequest struct { // semantics rather than a full entry list. Summary bool // Filter narrows the diff to a single change class. - // Filter behavior is layered on top of a provisional diff seam and is not yet - // a frozen daemon/API-ready contract. Filter SnapshotDiffFilter // Query filters which entries are considered. Query SnapshotQuery @@ -679,13 +653,7 @@ type SnapshotDiffSummary struct { Modified int } -// SnapshotDiffResult is the active but provisional result contract for -// Engine.SnapshotDiff. -// -// SummaryMode, MatchedEntryCount, and TotalEntryCount are provisional -// read-side semantics. They reflect the current summary-versus-detailed seam -// and filtering behavior, not a frozen daemon/API-ready diff contract. -// Read-side ownership and shape decisions are deferred to early v2.0. +// SnapshotDiffResult is the complete result contract for Engine.SnapshotDiff. type SnapshotDiffResult struct { BaseID string TargetID string diff --git a/internal/engine/contract_surface_clarity_test.go b/internal/engine/contract_surface_clarity_test.go index 3c0b4275..a345963d 100644 --- a/internal/engine/contract_surface_clarity_test.go +++ b/internal/engine/contract_surface_clarity_test.go @@ -9,17 +9,17 @@ func TestSnapshotQueryOrNilReturnsNilForEmptyQuery(t *testing.T) { } } -func TestEngineQueryToSnapshotQueryPreservesSinglePathAndPrefixShape(t *testing.T) { +func TestEngineQueryToSnapshotQueryPreservesRepeatedPathAndPrefixShape(t *testing.T) { min := int64(1) max := int64(8) got, err := engineQueryToSnapshotQuery(SnapshotQuery{ - Path: "docs/a.txt", - Prefix: "docs/", - Pattern: "*.txt", - Regex: "^docs/", - MinSize: &min, - MaxSize: &max, - Limit: 5, + Paths: []string{"docs/a.txt", "docs/b.txt"}, + Prefixes: []string{"docs/", "images/"}, + Pattern: "*.txt", + Regex: "^docs/", + MinSize: &min, + MaxSize: &max, + Limit: 5, }) if err != nil { t.Fatalf("engineQueryToSnapshotQuery: %v", err) @@ -27,14 +27,14 @@ func TestEngineQueryToSnapshotQueryPreservesSinglePathAndPrefixShape(t *testing. if got == nil { t.Fatal("engineQueryToSnapshotQuery: got nil") } - if len(got.ExactPaths) != 1 { - t.Fatalf("expected exactly one exact path, got %#v", got.ExactPaths) + if len(got.ExactPaths) != 2 { + t.Fatalf("expected two exact paths, got %#v", got.ExactPaths) } if _, ok := got.ExactPaths["docs/a.txt"]; !ok { t.Fatalf("expected exact path docs/a.txt, got %#v", got.ExactPaths) } - if len(got.Prefixes) != 1 || got.Prefixes[0] != "docs/" { - t.Fatalf("expected exactly one prefix docs/, got %#v", got.Prefixes) + if len(got.Prefixes) != 2 || got.Prefixes[0] != "docs/" || got.Prefixes[1] != "images/" { + t.Fatalf("expected repeated prefixes, got %#v", got.Prefixes) } } diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index de14c2aa..64979f22 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -435,8 +435,8 @@ func snapshotContractQuery() (engine.SnapshotQuery, time.Time, time.Time) { after := time.Unix(0, 0) before := time.Unix(1000, 0) q := engine.SnapshotQuery{ - Path: "p", - Prefix: "pre", + Paths: []string{"p", "q"}, + Prefixes: []string{"pre/", "other/"}, Pattern: "*.txt", Regex: ".*", MinSize: &min, diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index 3fd76730..12861a2d 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -7,9 +7,11 @@ import ( "encoding/hex" "errors" "fmt" + "path" "regexp" "strconv" "strings" + "time" "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/chunk" @@ -292,9 +294,14 @@ func projectSelectedSnapshotGraph(graph *catalog.SnapshotGraph, selected []catal ChildIDs: children, }) } - for _, rootID := range graph.RootIDs { - if _, ok := selectedIDs[rootID]; ok { - result.RootIDs = append(result.RootIDs, rootID) + // RootIDs are roots of the selected projection. A selected child whose + // existing parent was filtered out remains parent-aware metadata, but it is + // a top-level node for this projection. Historical missing parents are also + // top-level without inventing a parent edge. + for _, node := range result.Nodes { + _, parentSelected := selectedIDs[node.Snapshot.ParentID] + if node.ParentState != SnapshotParentPresent || !parentSelected { + result.RootIDs = append(result.RootIDs, node.Snapshot.ID) } } @@ -324,7 +331,7 @@ func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowReques CreatedAt: ref.CreatedAt, } var snapshotQ *snapshot.SnapshotQuery - if req.Query != (SnapshotQuery{}) { + if !isEmptySnapshotQuery(req.Query) { var err error snapshotQ, err = engineQueryToSnapshotQuery(req.Query) if err != nil { @@ -344,9 +351,9 @@ func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowReques files[i] = SnapshotFile{ StoredPath: entry.Path, LogicalFileID: entry.LogicalFileID, - Size: entry.Size.Int64, - Mode: uint32(entry.Mode.Int64), - ModTime: entry.MTime.Time, + Size: nullableInt64(entry.Size), + Mode: nullableInt64(entry.Mode), + ModTime: nullableTime(entry.MTime), } } return SnapshotShowResult{ @@ -433,11 +440,33 @@ func engineQueryToSnapshotQuery(q SnapshotQuery) (*snapshot.SnapshotQuery, error ModifiedAfter: q.ModifiedAfter, ModifiedBefore: q.ModifiedBefore, } - if q.Path != "" { - sq.ExactPaths = map[string]struct{}{q.Path: {}} + if len(q.Paths) > 0 { + sq.ExactPaths = make(map[string]struct{}, len(q.Paths)) + for _, rawPath := range q.Paths { + normalized, err := snapshot.NormalizeSnapshotPath(rawPath) + if err != nil { + return nil, fmt.Errorf("invalid snapshot query path %q: %w", rawPath, err) + } + sq.ExactPaths[normalized] = struct{}{} + } } - if q.Prefix != "" { - sq.Prefixes = []string{q.Prefix} + if len(q.Prefixes) > 0 { + sq.Prefixes = make([]string, 0, len(q.Prefixes)) + for _, rawPrefix := range q.Prefixes { + normalized, err := snapshot.NormalizeSnapshotPath(rawPrefix) + if err != nil { + return nil, fmt.Errorf("invalid snapshot query prefix %q: %w", rawPrefix, err) + } + if !strings.HasSuffix(normalized, "/") { + return nil, fmt.Errorf("invalid snapshot query prefix %q: must end with '/'", rawPrefix) + } + sq.Prefixes = append(sq.Prefixes, normalized) + } + } + if q.Pattern != "" { + if _, err := path.Match(q.Pattern, ""); err != nil { + return nil, fmt.Errorf("invalid snapshot query pattern %q: %w", q.Pattern, err) + } } if q.Regex != "" { compiled, err := regexp.Compile(q.Regex) @@ -446,5 +475,30 @@ func engineQueryToSnapshotQuery(q SnapshotQuery) (*snapshot.SnapshotQuery, error } sq.Regex = compiled } + if (q.MinSize != nil && *q.MinSize < 0) || (q.MaxSize != nil && *q.MaxSize < 0) { + return nil, fmt.Errorf("invalid snapshot query size range") + } + if q.MinSize != nil && q.MaxSize != nil && *q.MinSize > *q.MaxSize { + return nil, fmt.Errorf("invalid snapshot query size range: minimum exceeds maximum") + } + if q.ModifiedAfter != nil && q.ModifiedBefore != nil && q.ModifiedAfter.After(*q.ModifiedBefore) { + return nil, fmt.Errorf("invalid snapshot query time range: after exceeds before") + } return sq, nil } + +func nullableInt64(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + result := value.Int64 + return &result +} + +func nullableTime(value sql.NullTime) *time.Time { + if !value.Valid { + return nil + } + result := value.Time + return &result +} diff --git a/internal/engine/default_engine_routing_helpers.go b/internal/engine/default_engine_routing_helpers.go index 5a1d27a1..fa656edf 100644 --- a/internal/engine/default_engine_routing_helpers.go +++ b/internal/engine/default_engine_routing_helpers.go @@ -19,7 +19,7 @@ import ( ) func isSnapshotDiffSummaryFastPath(req SnapshotDiffRequest) bool { - return req.Summary && req.Filter == "" && req.Query == (SnapshotQuery{}) + return req.Summary && req.Filter == "" && isEmptySnapshotQuery(req.Query) } func (e *DefaultEngine) snapshotDiffSummaryFastPath(ctx context.Context, req SnapshotDiffRequest) (SnapshotDiffResult, error) { @@ -58,19 +58,27 @@ func (e *DefaultEngine) snapshotDiffDetailed(ctx context.Context, req SnapshotDi if !snapshotDiffEntryMatchesFilter(diffType, req.Filter) { continue } - entries = append(entries, SnapshotDiffEntry{StoredPath: entry.Path, Change: SnapshotDiffChange(diffType)}) + entries = append(entries, SnapshotDiffEntry{ + StoredPath: entry.Path, Change: SnapshotDiffChange(diffType), + BaseLogicalID: nullableInt64(entry.BaseLogicalID), TargetLogicalID: nullableInt64(entry.TargetLogicalID), + }) addSnapshotDiffSummaryEntry(&summary, diffType) } return buildSnapshotDiffResult(req, entries, summary, len(raw.Entries)), nil } func snapshotQueryOrNil(q SnapshotQuery) (*snapshot.SnapshotQuery, error) { - if q == (SnapshotQuery{}) { + if isEmptySnapshotQuery(q) { return nil, nil } return engineQueryToSnapshotQuery(q) } +func isEmptySnapshotQuery(q SnapshotQuery) bool { + return len(q.Paths) == 0 && len(q.Prefixes) == 0 && q.Pattern == "" && q.Regex == "" && + q.MinSize == nil && q.MaxSize == nil && q.ModifiedAfter == nil && q.ModifiedBefore == nil && q.Limit == 0 +} + func snapshotDiffEntryMatchesFilter(diffType string, filter SnapshotDiffFilter) bool { return filter == "" || SnapshotDiffFilter(diffType) == filter } diff --git a/internal/engine/snapshot_selector_backend_contract_test.go b/internal/engine/snapshot_selector_backend_contract_test.go index e5a80bc2..b35d1701 100644 --- a/internal/engine/snapshot_selector_backend_contract_test.go +++ b/internal/engine/snapshot_selector_backend_contract_test.go @@ -41,10 +41,17 @@ func TestEngineSnapshotSelectorsAcrossBackends(t *testing.T) { if got, want := tree.Graph.Nodes[0].ChildIDs, []string{"snap-base"}; !reflect.DeepEqual(got, want) { t.Fatalf("tree root children: got %v want %v", got, want) } + filteredTree, err := fixture.engine.SnapshotList(context.Background(), engine.SnapshotListRequest{Label: "target", Tree: true}) + if err != nil || filteredTree.Graph == nil || !reflect.DeepEqual(filteredTree.Graph.RootIDs, []string{"snap-target"}) { + t.Fatalf("filtered tree projection: got (%+v, %v)", filteredTree, err) + } + if filteredTree.Graph.Nodes[0].ParentState != engine.SnapshotParentPresent { + t.Fatalf("filtered tree lost existing-parent state: %+v", filteredTree.Graph.Nodes[0]) + } query := engine.SnapshotQuery{ - Path: "docs/added.txt", - Prefix: "docs/", + Paths: []string{"docs/added.txt"}, + Prefixes: []string{"docs/"}, Pattern: "docs/*.txt", Regex: "added\\.txt$", MinSize: int64Pointer(10), @@ -89,6 +96,12 @@ func TestEngineSnapshotSelectorErrorsAcrossBackends(t *testing.T) { if _, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target", Query: invalid}); err == nil || !strings.Contains(err.Error(), "invalid snapshot query regex") { t.Fatalf("invalid SnapshotDiff regex: %v", err) } + if _, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "snap-target", Query: engine.SnapshotQuery{Paths: []string{" docs/common.txt"}}}); err == nil || !strings.Contains(err.Error(), "leading or trailing whitespace") { + t.Fatalf("invalid SnapshotShow path: %v", err) + } + if _, err := fixture.engine.SnapshotDiff(context.Background(), engine.SnapshotDiffRequest{BaseID: "snap-base", TargetID: "snap-target", Query: engine.SnapshotQuery{Prefixes: []string{"docs"}}}); err == nil || !strings.Contains(err.Error(), "must end with '/'") { + t.Fatalf("invalid SnapshotDiff prefix: %v", err) + } cancelled, cancel := context.WithCancel(context.Background()) cancel() if _, err := fixture.engine.SnapshotList(cancelled, engine.SnapshotListRequest{}); !errors.Is(err, context.Canceled) { From f3eb713194379a7978f80abc61e4b797ae98a738 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 14:46:56 +0200 Subject: [PATCH 15/27] Route inspect stats and verify through engine --- .../legacy_observability_verify_seams_test.go | 79 +++++ cmd/coldkeep/main.go | 318 ++++++------------ cmd/coldkeep/observability_engine_adapters.go | 36 ++ ...bservability_verify_engine_routing_test.go | 162 +++++++++ cmd/coldkeep/stored_path_engine_adapters.go | 8 + .../stored_path_engine_routing_test.go | 24 ++ docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...hase14-inspect-stats-and-verify-routing.md | 58 ++++ 8 files changed, 479 insertions(+), 212 deletions(-) create mode 100644 cmd/coldkeep/legacy_observability_verify_seams_test.go create mode 100644 cmd/coldkeep/observability_verify_engine_routing_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase14-inspect-stats-and-verify-routing.md diff --git a/cmd/coldkeep/legacy_observability_verify_seams_test.go b/cmd/coldkeep/legacy_observability_verify_seams_test.go new file mode 100644 index 00000000..525b9929 --- /dev/null +++ b/cmd/coldkeep/legacy_observability_verify_seams_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" + "github.com/franchoy/coldkeep/internal/verify" +) + +type verifyOutputSummary struct { + BlocksChecked int64 + PhysicalHashChecked int64 + CompressedHashChecked int64 + LogicalHashChecked int64 + CompressedBlocksChecked int64 +} + +var verifyCommandPhase = func(dbconn *sql.DB, target string, fileID int, level verify.VerifyLevel) error { + eng, err := engine.New(engine.Config{DB: dbconn}) + if err != nil { + return err + } + _, err = eng.Verify(context.Background(), engine.VerifyRequest{Target: target, FileID: fileID, Level: verifyLevelToString(level)}) + return err +} + +var verifySummaryPhase = func(_ *sql.DB, _ string, _ int64) (verifyOutputSummary, error) { + return verifyOutputSummary{}, nil +} + +var productionVerifyCommandEngine = newVerifyCommandEngine + +func init() { + newVerifyCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return legacyVerifyTestEngine{db: sgctx.DB}, nil + } +} + +type legacyVerifyTestEngine struct { + engine.Engine + db *sql.DB +} + +func (e legacyVerifyTestEngine) Verify(_ context.Context, req engine.VerifyRequest) (engine.VerifyResult, error) { + level, err := legacyVerifyLevel(req.Level) + if err != nil { + return engine.VerifyResult{}, err + } + if err := verifyCommandPhase(e.db, req.Target, req.FileID, level); err != nil { + return engine.VerifyResult{}, err + } + summary, err := verifySummaryPhase(e.db, req.Target, int64(req.FileID)) + if err != nil { + return engine.VerifyResult{}, err + } + return engine.VerifyResult{ + BlocksChecked: summary.BlocksChecked, PhysicalHashChecked: summary.PhysicalHashChecked, + CompressedHashChecked: summary.CompressedHashChecked, LogicalHashChecked: summary.LogicalHashChecked, + CompressedBlocksChecked: summary.CompressedBlocksChecked, + }, nil +} + +func legacyVerifyLevel(value string) (verify.VerifyLevel, error) { + switch value { + case "fast": + return verify.VerifyFast, nil + case "", "standard": + return verify.VerifyStandard, nil + case "full": + return verify.VerifyFull, nil + case "deep": + return verify.VerifyDeep, nil + default: + return 0, fmt.Errorf("unknown verify level %q", value) + } +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 689d04dd..3c8e9f3a 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -151,14 +151,6 @@ type cliRuntime struct { now func() time.Time } -type verifyOutputSummary struct { - BlocksChecked int64 - PhysicalHashChecked int64 - CompressedHashChecked int64 - LogicalHashChecked int64 - CompressedBlocksChecked int64 -} - type verifyFailureDetails struct { Stage string Block *int64 @@ -329,7 +321,7 @@ var runGCPhase = func(dryRun bool, containersDir string) (maintenance.GCResult, if err != nil { return maintenance.GCResult{}, err } - defer func() { _ = sgctx.DB.Close() }() + defer func() { _ = sgctx.Close() }() eng, err := engine.New(engine.Config{ DB: sgctx.DB, @@ -387,7 +379,6 @@ func snapshotMetaToSnapshot(m engine.SnapshotMeta) snapshot.Snapshot { return s } -var newObservabilityServicePhase = observability.NewService var runObservabilityStatsPhase = func(opts observability.StatsOptions) (*observability.StatsResult, error) { sgctx, err := loadDefaultStorageContextPhase() if err != nil { @@ -395,10 +386,7 @@ var runObservabilityStatsPhase = func(opts observability.StatsOptions) (*observa } defer func() { _ = sgctx.DB.Close() }() - eng, err := engine.New(engine.Config{ - DB: sgctx.DB, - ContainerDir: sgctx.EffectiveContainerDir(), - }) + eng, err := newObservabilityCommandEngine(sgctx) if err != nil { return nil, err } @@ -417,43 +405,36 @@ var runObservabilityStatsPhase = func(opts observability.StatsOptions) (*observa return statsResultFromEngine(result), nil } var runObservabilityInspectPhase = func(entity observability.EntityType, id string, opts observability.InspectOptions) (*observability.InspectResult, error) { - // Engine.Inspect is active, while this production CLI path intentionally - // still calls the observability service directly. Active method presence does - // not prove complete read-side workflow ownership; early v2.0 owns any - // remaining ownership decision. sgctx, err := loadDefaultStorageContextPhase() if err != nil { return nil, err } - defer func() { _ = sgctx.DB.Close() }() + defer func() { _ = sgctx.Close() }() - svc, err := newObservabilityServicePhase(sgctx.DB) + eng, err := newObservabilityCommandEngine(sgctx) if err != nil { return nil, err } - - r, err := svc.Inspect(context.Background(), entity, id, opts) - if err != nil { - return nil, err + result, err := eng.Inspect(context.Background(), engine.InspectRequest{ + Entity: engine.InspectEntity(entity), EntityID: id, + Options: engine.InspectOptions{ + Deep: opts.Deep, Relations: opts.Relations, Reverse: opts.Reverse, + Limit: opts.Limit, IncludeTrace: opts.Trace.Enabled, + }, + }) + if traceErr := replayEngineTrace(opts.Trace, result.Trace); traceErr != nil { + return nil, traceErr } - - return r, nil -} -var verifyCommandPhase = func(dbconn *sql.DB, target string, fileID int, level verify.VerifyLevel) error { - eng, err := engine.New(engine.Config{DB: dbconn, ContainerDir: container.ContainersDir}) if err != nil { - return err + return nil, err } - _, err = eng.Verify(context.Background(), engine.VerifyRequest{ - Level: verifyLevelToString(level), - Target: target, - FileID: fileID, - }) - return err -} -var verifySummaryPhase = func(dbconn *sql.DB, target string, fileID int64) (verifyOutputSummary, error) { - return collectVerifyOutputSummary(dbconn, target, fileID) + return inspectResultFromEngine(result) } + +// Simulation and benchmark commands are explicitly classified tooling. Their +// isolated simulated repositories retain the observability service seam; no +// production inspect, stats, or verify command uses it. +var newObservabilityServicePhase = observability.NewService var runChunkerBenchmarkPhase = runChunkerBenchmark var runCoreBenchmarkPhase = runCoreBenchmark var runBenchmarkDeterminismPhase = validateBenchmarkDeterminism @@ -1050,92 +1031,6 @@ func extractVerifyFailureDetails(err error) (verifyFailureDetails, bool) { return details, true } -func countVerifySummaryForSystem(dbconn *sql.DB) (verifyOutputSummary, error) { - var s verifyOutputSummary - - if err := dbconn.QueryRow(` - SELECT - COUNT(*), - COALESCE(SUM(CASE WHEN physical_hash IS NOT NULL AND length(physical_hash) > 0 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN compressed_hash IS NOT NULL AND length(compressed_hash) > 0 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN block_hash IS NOT NULL AND length(block_hash) > 0 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN lower(trim(COALESCE(compression_codec, 'none'))) != 'none' THEN 1 ELSE 0 END), 0) - FROM storage_blocks sb - JOIN container c ON c.id = sb.container_id - WHERE c.quarantine = FALSE - `).Scan(&s.BlocksChecked, &s.PhysicalHashChecked, &s.CompressedHashChecked, &s.LogicalHashChecked, &s.CompressedBlocksChecked); err != nil { - return verifyOutputSummary{}, err - } - - var legacyBlocks int64 - if err := dbconn.QueryRow(` - SELECT COUNT(*) - FROM blocks b - JOIN chunk c ON c.id = b.chunk_id - WHERE c.status = 'COMPLETED' - AND NOT EXISTS (SELECT 1 FROM chunk_block_refs r WHERE r.chunk_id = c.id) - `).Scan(&legacyBlocks); err != nil { - return verifyOutputSummary{}, err - } - - s.BlocksChecked += legacyBlocks - return s, nil -} - -func countVerifySummaryForFile(dbconn *sql.DB, fileID int64) (verifyOutputSummary, error) { - var s verifyOutputSummary - - if err := dbconn.QueryRow(` - WITH target_blocks AS ( - SELECT DISTINCT sb.id, sb.physical_hash, sb.compressed_hash, sb.block_hash, sb.compression_codec - FROM file_chunk fc - JOIN chunk_block_refs r ON r.chunk_id = fc.chunk_id - JOIN storage_blocks sb ON sb.id = r.block_id - JOIN container c ON c.id = sb.container_id - WHERE fc.logical_file_id = $1 - AND c.quarantine = FALSE - ) - SELECT - COUNT(*), - COALESCE(SUM(CASE WHEN physical_hash IS NOT NULL AND length(physical_hash) > 0 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN compressed_hash IS NOT NULL AND length(compressed_hash) > 0 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN block_hash IS NOT NULL AND length(block_hash) > 0 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN lower(trim(COALESCE(compression_codec, 'none'))) != 'none' THEN 1 ELSE 0 END), 0) - FROM target_blocks - `, fileID).Scan(&s.BlocksChecked, &s.PhysicalHashChecked, &s.CompressedHashChecked, &s.LogicalHashChecked, &s.CompressedBlocksChecked); err != nil { - return verifyOutputSummary{}, err - } - - var legacyBlocks int64 - if err := dbconn.QueryRow(` - SELECT COUNT(*) - FROM file_chunk fc - JOIN blocks b ON b.chunk_id = fc.chunk_id - WHERE fc.logical_file_id = $1 - AND NOT EXISTS (SELECT 1 FROM chunk_block_refs r WHERE r.chunk_id = b.chunk_id) - `, fileID).Scan(&legacyBlocks); err != nil { - return verifyOutputSummary{}, err - } - - s.BlocksChecked += legacyBlocks - return s, nil -} - -func collectVerifyOutputSummary(dbconn *sql.DB, target string, fileID int64) (verifyOutputSummary, error) { - if dbconn == nil { - return verifyOutputSummary{}, fmt.Errorf("verify summary DB connection is nil") - } - - switch target { - case "system": - return countVerifySummaryForSystem(dbconn) - case "file": - return countVerifySummaryForFile(dbconn, fileID) - default: - return verifyOutputSummary{}, fmt.Errorf("unknown verify target: %s", target) - } -} - func runVersionCommand(mode cliOutputMode) error { if mode == outputModeJSON { payload := map[string]any{ @@ -2382,7 +2277,7 @@ func runInspectCommand(parsed parsedCommandLine, outputMode cliOutputMode) error r, err := runObservabilityInspectPhase(entityType, entityID, opts) if err != nil { - if errors.Is(err, observability.ErrNotFound) || errors.Is(err, sql.ErrNoRows) { + if engine.IsCode(err, engine.ErrorNotFound) || errors.Is(err, observability.ErrNotFound) || errors.Is(err, sql.ErrNoRows) { return observabilityErrorf(exitGeneral, "NOT_FOUND", "%s %s not found", entityLabel, entityID) } return observabilityWrappedError(exitGeneral, "INTERNAL", "inspect failed", err) @@ -2754,56 +2649,16 @@ func runVerifyCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } target := parsed.positionals[0] + fileID := 0 switch target { case "system": - sgctx, err := loadDefaultStorageContextPhase() - if err != nil { - return fmt.Errorf("load storage context: %w", err) - } - defer func() { _ = sgctx.Close() }() - - verifyErr := verifyCommandPhase(sgctx.DB, target, 0, verifyLevel) - if verifyErr != nil { - return verifyError(verifyErr) - } - summary, err := verifySummaryPhase(sgctx.DB, target, 0) - if err != nil { - return fmt.Errorf("collect verify summary: %w", err) - } - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "verify", - "target": target, - "level": verifyLevelToString(verifyLevel), - "verify": "ok", - "blocks_checked": summary.BlocksChecked, - "physical_hash_checked": summary.PhysicalHashChecked, - "compressed_hash_checked": summary.CompressedHashChecked, - "logical_hash_checked": summary.LogicalHashChecked, - "compressed_blocks_checked": summary.CompressedBlocksChecked, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil - } - fmt.Println("verify ok") - fmt.Printf("blocks_checked: %d\n", summary.BlocksChecked) - fmt.Printf("physical_hash_checked: %d\n", summary.PhysicalHashChecked) - fmt.Printf("compressed_hash_checked: %d\n", summary.CompressedHashChecked) - fmt.Printf("logical_hash_checked: %d\n", summary.LogicalHashChecked) - fmt.Printf("compressed_blocks_checked: %d\n", summary.CompressedBlocksChecked) - if outputMode == outputModeText { - fmt.Printf("Hint: %s\n", doctorOperationalHint) - } - return nil case "file": if len(parsed.positionals) < 2 || len(parsed.positionals) > 3 { return usageErrorf("Usage: coldkeep verify file [--fast|--standard|--full|--deep]") } fileIDText := parsed.positionals[1] - fileID, err := strconv.Atoi(fileIDText) + fileID, err = strconv.Atoi(fileIDText) if err != nil { if errors.Is(err, strconv.ErrRange) { return usageErrorf("Invalid fileID: value %s exceeds platform int range", fileIDText) @@ -2813,52 +2668,60 @@ func runVerifyCommand(parsed parsedCommandLine, outputMode cliOutputMode) error if fileID <= 0 { return usageErrorf("Invalid fileID: must be a positive integer") } + default: + return usageErrorf("Unknown target for verify: %s (expected 'system' or 'file ')", target) + } - sgctx, err := loadDefaultStorageContextPhase() - if err != nil { - return fmt.Errorf("load storage context: %w", err) + sgctx, err := loadDefaultStorageContextPhase() + if err != nil { + return fmt.Errorf("load storage context: %w", err) + } + defer func() { _ = sgctx.Close() }() + eng, err := newVerifyCommandEngine(sgctx) + if err != nil { + return err + } + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + result, err := eng.Verify(ctx, engine.VerifyRequest{ + Target: target, FileID: fileID, Level: verifyLevelToString(verifyLevel), + }) + if err != nil { + // Before summary ownership moved into Engine.Verify, a failure collecting + // the post-verification counters was a general command failure rather than + // a verification-integrity failure. Preserve that exit classification. + if engine.IsCode(err, engine.ErrorOperationFailed) && strings.Contains(err.Error(), "collect verify summary") { + return err } - defer func() { _ = sgctx.Close() }() + return verifyError(err) + } + return renderVerifySuccess(outputMode, target, fileID, verifyLevel, result) +} - verifyErr := verifyCommandPhase(sgctx.DB, target, fileID, verifyLevel) - if verifyErr != nil { - return verifyError(verifyErr) - } - summary, err := verifySummaryPhase(sgctx.DB, target, int64(fileID)) - if err != nil { - return fmt.Errorf("collect verify summary: %w", err) - } - if outputMode == outputModeJSON { - payload := map[string]any{ - "status": "ok", - "command": "verify", - "target": target, - "file_id": fileID, - "level": verifyLevelToString(verifyLevel), - "verify": "ok", - "blocks_checked": summary.BlocksChecked, - "physical_hash_checked": summary.PhysicalHashChecked, - "compressed_hash_checked": summary.CompressedHashChecked, - "logical_hash_checked": summary.LogicalHashChecked, - "compressed_blocks_checked": summary.CompressedBlocksChecked, - } - encoded, _ := json.Marshal(payload) - fmt.Println(string(encoded)) - return nil +func renderVerifySuccess(outputMode cliOutputMode, target string, fileID int, level verify.VerifyLevel, result engine.VerifyResult) error { + if outputMode == outputModeJSON { + payload := map[string]any{ + "status": "ok", "command": "verify", "target": target, + "level": verifyLevelToString(level), "verify": "ok", + "blocks_checked": result.BlocksChecked, "physical_hash_checked": result.PhysicalHashChecked, + "compressed_hash_checked": result.CompressedHashChecked, "logical_hash_checked": result.LogicalHashChecked, + "compressed_blocks_checked": result.CompressedBlocksChecked, } - fmt.Println("verify ok") - fmt.Printf("blocks_checked: %d\n", summary.BlocksChecked) - fmt.Printf("physical_hash_checked: %d\n", summary.PhysicalHashChecked) - fmt.Printf("compressed_hash_checked: %d\n", summary.CompressedHashChecked) - fmt.Printf("logical_hash_checked: %d\n", summary.LogicalHashChecked) - fmt.Printf("compressed_blocks_checked: %d\n", summary.CompressedBlocksChecked) - if outputMode == outputModeText { - fmt.Printf("Hint: %s\n", doctorOperationalHint) + if target == "file" { + payload["file_id"] = fileID } + encoded, _ := json.Marshal(payload) + fmt.Println(string(encoded)) return nil - default: - return usageErrorf("Unknown target for verify: %s (expected 'system' or 'file ')", target) } + fmt.Println("verify ok") + fmt.Printf("blocks_checked: %d\n", result.BlocksChecked) + fmt.Printf("physical_hash_checked: %d\n", result.PhysicalHashChecked) + fmt.Printf("compressed_hash_checked: %d\n", result.CompressedHashChecked) + fmt.Printf("logical_hash_checked: %d\n", result.LogicalHashChecked) + fmt.Printf("compressed_blocks_checked: %d\n", result.CompressedBlocksChecked) + fmt.Printf("Hint: %s\n", doctorOperationalHint) + return nil } // runDoctorCommand implements the doctor corrective recovery command. @@ -3618,6 +3481,41 @@ type benchmarkDiagnosticVerification struct { SnapshotReachabilityIssues int64 `json:"snapshot_reachability_issues"` } +// countBenchmarkVerificationTotals is diagnostic tooling for isolated +// benchmark final-state comparison. Production verify commands use +// Engine.Verify and never call this direct metadata query. +func countBenchmarkVerificationTotals(dbconn *sql.DB) (benchmarkDiagnosticVerification, error) { + var result benchmarkDiagnosticVerification + if err := dbconn.QueryRow(` + SELECT + COUNT(*), + COALESCE(SUM(CASE WHEN physical_hash IS NOT NULL AND length(physical_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN compressed_hash IS NOT NULL AND length(compressed_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN block_hash IS NOT NULL AND length(block_hash) > 0 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN lower(trim(COALESCE(compression_codec, 'none'))) != 'none' THEN 1 ELSE 0 END), 0) + FROM storage_blocks sb + JOIN container c ON c.id = sb.container_id + WHERE c.quarantine = FALSE + `).Scan( + &result.BlocksChecked, &result.PhysicalHashesChecked, &result.CompressedHashesChecked, + &result.LogicalHashesChecked, &result.CompressedBlocksChecked, + ); err != nil { + return benchmarkDiagnosticVerification{}, err + } + var legacyBlocks int64 + if err := dbconn.QueryRow(` + SELECT COUNT(*) + FROM blocks b + JOIN chunk c ON c.id = b.chunk_id + WHERE c.status = 'COMPLETED' + AND NOT EXISTS (SELECT 1 FROM chunk_block_refs r WHERE r.chunk_id = c.id) + `).Scan(&legacyBlocks); err != nil { + return benchmarkDiagnosticVerification{}, err + } + result.BlocksChecked += legacyBlocks + return result, nil +} + type benchmarkDiagnosticPhysical struct { ContainerCount int64 `json:"container_count"` StorageBlockCount int64 `json:"storage_block_count"` @@ -4043,7 +3941,7 @@ func buildBenchmarkDiagnosticFinalState( if err != nil { return benchmarkDiagnosticFinalState{}, fmt.Errorf("capture GC reachability totals: %w", err) } - verifyTotals, err := countVerifySummaryForSystem(dbconn) + verifyTotals, err := countBenchmarkVerificationTotals(dbconn) if err != nil { return benchmarkDiagnosticFinalState{}, fmt.Errorf("capture verification totals: %w", err) } @@ -4079,8 +3977,8 @@ func buildBenchmarkDiagnosticFinalState( RetainedDeadBytes: gcPlan.Summary.RetainedDeadBytesDueToPackedBlocks, }, Verification: benchmarkDiagnosticVerification{ - BlocksChecked: verifyTotals.BlocksChecked, PhysicalHashesChecked: verifyTotals.PhysicalHashChecked, - CompressedHashesChecked: verifyTotals.CompressedHashChecked, LogicalHashesChecked: verifyTotals.LogicalHashChecked, + BlocksChecked: verifyTotals.BlocksChecked, PhysicalHashesChecked: verifyTotals.PhysicalHashesChecked, + CompressedHashesChecked: verifyTotals.CompressedHashesChecked, LogicalHashesChecked: verifyTotals.LogicalHashesChecked, CompressedBlocksChecked: verifyTotals.CompressedBlocksChecked, PhysicalFileIssues: physicalAudit.OrphanPhysicalFileRows + physicalAudit.LogicalRefCountMismatches + physicalAudit.NegativeLogicalRefCounts, SnapshotMembershipRows: snapshotAudit.SnapshotFileRows, diff --git a/cmd/coldkeep/observability_engine_adapters.go b/cmd/coldkeep/observability_engine_adapters.go index c51b7699..b837d82d 100644 --- a/cmd/coldkeep/observability_engine_adapters.go +++ b/cmd/coldkeep/observability_engine_adapters.go @@ -102,6 +102,42 @@ func statsResultFromEngine(input engine.StatsResult) *observability.StatsResult } } +func inspectResultFromEngine(input engine.InspectResult) (*observability.InspectResult, error) { + summary, err := engineValuesToAny(input.Summary) + if err != nil { + return nil, fmt.Errorf("project inspect summary: %w", err) + } + metadata, err := engineValuesToAny(input.Metadata) + if err != nil { + return nil, fmt.Errorf("project inspect metadata: %w", err) + } + relations := make([]observability.Relation, len(input.Relations)) + for i, item := range input.Relations { + relationMetadata, err := engineValuesToAny(item.Metadata) + if err != nil { + return nil, fmt.Errorf("project inspect relation %d: %w", i, err) + } + relations[i] = observability.Relation{ + Type: item.Type, Direction: observability.RelationDirection(item.Direction), + TargetType: observability.EntityType(item.TargetType), TargetID: item.TargetID, + Metadata: relationMetadata, + } + } + warnings := make([]observability.ObservationWarning, len(input.Warnings)) + for i, item := range input.Warnings { + warnings[i] = observability.ObservationWarning{Code: item.Code, Message: item.Message} + } + return &observability.InspectResult{ + GeneratedAtUTC: input.GeneratedAtUTC, + EntityType: observability.EntityType(input.Entity), + EntityID: input.EntityID, + Summary: summary, + Metadata: metadata, + Relations: relations, + Warnings: warnings, + }, nil +} + func replayEngineTrace(options observability.TraceOptions, events []engine.TraceEvent) error { if !options.Enabled || options.Sink == nil { return nil diff --git a/cmd/coldkeep/observability_verify_engine_routing_test.go b/cmd/coldkeep/observability_verify_engine_routing_test.go new file mode 100644 index 00000000..f3efd45a --- /dev/null +++ b/cmd/coldkeep/observability_verify_engine_routing_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" +) + +var productionStatsEnginePhaseForTest = runObservabilityStatsPhase +var productionInspectEnginePhaseForTest = runObservabilityInspectPhase + +func installObservabilityEngineStub(t *testing.T, stub engine.Engine) { + t.Helper() + dbconn := openSnapshotRoutingDB(t) + originalLoad := loadDefaultStorageContextPhase + originalFactory := newObservabilityCommandEngine + originalStats := runObservabilityStatsPhase + originalInspect := runObservabilityInspectPhase + t.Cleanup(func() { + loadDefaultStorageContextPhase = originalLoad + newObservabilityCommandEngine = originalFactory + runObservabilityStatsPhase = originalStats + runObservabilityInspectPhase = originalInspect + }) + loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { + return storage.StorageContext{DB: dbconn}, nil + } + newObservabilityCommandEngine = func(storage.StorageContext) (engine.Engine, error) { return stub, nil } + runObservabilityStatsPhase = productionStatsEnginePhaseForTest + runObservabilityInspectPhase = productionInspectEnginePhaseForTest +} + +func TestStatsAndInspectRouteThroughTypedEngineResults(t *testing.T) { + statsCalls, inspectCalls := 0, 0 + installObservabilityEngineStub(t, stubCommandEngine{ + statsFunc: func(_ context.Context, req engine.StatsRequest) (engine.StatsResult, error) { + statsCalls++ + if !req.IncludeContainers || req.IncludeTrace { + t.Fatalf("stats request = %+v", req) + } + return engine.StatsResult{Logical: engine.StatsLogical{TotalFiles: 7}}, nil + }, + inspectFunc: func(_ context.Context, req engine.InspectRequest) (engine.InspectResult, error) { + inspectCalls++ + wantOptions := engine.InspectOptions{Deep: true, Relations: true, Reverse: true, Limit: 9} + if req.Entity != engine.InspectFile || req.EntityID != "42" || !reflect.DeepEqual(req.Options, wantOptions) { + t.Fatalf("inspect request = %+v", req) + } + return engine.InspectResult{ + Entity: engine.InspectFile, EntityID: "42", + Summary: map[string]engine.Value{"exact": {Kind: engine.ValueInteger, Integer: "9007199254740993"}}, + }, nil + }, + }) + + statsJSON := captureStdout(t, func() { + if err := runStatsCommand(parsedCommandLine{method: "stats", flags: map[string][]string{"containers": {""}, "output": {"json"}}}, outputModeJSON); err != nil { + t.Fatalf("stats: %v", err) + } + }) + if !strings.Contains(statsJSON, `"total_files":7`) { + t.Fatalf("stats JSON = %s", statsJSON) + } + inspectJSON := captureStdout(t, func() { + if err := runInspectCommand(parsedCommandLine{ + method: "inspect", positionals: []string{"file", "42"}, + flags: map[string][]string{"deep": {""}, "relations": {""}, "reverse": {""}, "limit": {"9"}, "output": {"json"}}, + }, outputModeJSON); err != nil { + t.Fatalf("inspect: %v", err) + } + }) + if !strings.Contains(inspectJSON, `"exact":9007199254740993`) { + t.Fatalf("inspect exact integer JSON = %s", inspectJSON) + } + if statsCalls != 1 || inspectCalls != 1 { + t.Fatalf("calls stats=%d inspect=%d", statsCalls, inspectCalls) + } +} + +func TestVerifyUsesOneEngineOperationAndItsSummary(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + originalLoad := loadDefaultStorageContextPhase + originalFactory := newVerifyCommandEngine + t.Cleanup(func() { + loadDefaultStorageContextPhase = originalLoad + newVerifyCommandEngine = originalFactory + }) + loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { + return storage.StorageContext{DB: dbconn}, nil + } + calls := 0 + newVerifyCommandEngine = func(storage.StorageContext) (engine.Engine, error) { + return stubCommandEngine{verifyFunc: func(_ context.Context, req engine.VerifyRequest) (engine.VerifyResult, error) { + calls++ + if req.Target != "file" || req.FileID != 42 || req.Level != "deep" { + t.Fatalf("verify request = %+v", req) + } + return engine.VerifyResult{BlocksChecked: 5, PhysicalHashChecked: 4, CompressedHashChecked: 3, LogicalHashChecked: 2, CompressedBlocksChecked: 1}, nil + }}, nil + } + + output := captureStdout(t, func() { + if err := runVerifyCommand(parsedCommandLine{ + method: "verify", positionals: []string{"file", "42"}, + flags: map[string][]string{"deep": {""}, "output": {"json"}}, + }, outputModeJSON); err != nil { + t.Fatalf("verify: %v", err) + } + }) + if calls != 1 { + t.Fatalf("Verify calls = %d, want 1", calls) + } + var payload map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(output)), &payload); err != nil { + t.Fatalf("verify JSON: %v (%s)", err, output) + } + if payload["blocks_checked"] != float64(5) || payload["compressed_blocks_checked"] != float64(1) { + t.Fatalf("verify summary = %#v", payload) + } +} + +func TestVerifySummaryFailureRetainsGeneralExitClassification(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + originalLoad := loadDefaultStorageContextPhase + originalFactory := newVerifyCommandEngine + t.Cleanup(func() { + loadDefaultStorageContextPhase = originalLoad + newVerifyCommandEngine = originalFactory + }) + loadDefaultStorageContextPhase = func() (storage.StorageContext, error) { + return storage.StorageContext{DB: dbconn}, nil + } + newVerifyCommandEngine = func(storage.StorageContext) (engine.Engine, error) { + return stubCommandEngine{verifyFunc: func(context.Context, engine.VerifyRequest) (engine.VerifyResult, error) { + return engine.VerifyResult{}, engine.NewError(engine.ErrorOperationFailed, "verify", "collect verify summary: read failed", "", nil) + }}, nil + } + err := runVerifyCommand(parsedCommandLine{method: "verify", positionals: []string{"system"}, flags: map[string][]string{}}, outputModeText) + if err == nil || classifyExitCode(err) != exitGeneral { + t.Fatalf("summary error = %v, exit=%d", err, classifyExitCode(err)) + } +} + +func TestInspectEngineNotFoundPreservesCLIProjection(t *testing.T) { + installObservabilityEngineStub(t, stubCommandEngine{ + inspectFunc: func(context.Context, engine.InspectRequest) (engine.InspectResult, error) { + return engine.InspectResult{}, engine.NewError(engine.ErrorNotFound, "inspect", "missing", "", nil) + }, + }) + err := runInspectCommand(parsedCommandLine{method: "inspect", positionals: []string{"snapshot", "missing"}, flags: map[string][]string{}}, outputModeText) + if err == nil || !strings.Contains(err.Error(), "snapshot missing not found") { + t.Fatalf("not-found projection = %v", err) + } + if got := publicErrorCode(err, classifyExitCode(err)); got != "NOT_FOUND" { + t.Fatalf("public code = %q", got) + } +} diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index 103368a4..77ecaa33 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -44,6 +44,14 @@ var newSnapshotReadCommandEngine = func(sgctx storage.StorageContext) (engine.En return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) } +var newVerifyCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} + +var newObservabilityCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 8d8e52ab..25c97f1a 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -31,6 +31,9 @@ type stubCommandEngine struct { searchFilesFunc func(context.Context, engine.SearchFilesRequest) (engine.SearchFilesResult, error) getConfigurationFunc func(context.Context, engine.GetConfigurationRequest) (engine.GetConfigurationResult, error) setConfigurationFunc func(context.Context, engine.SetConfigurationRequest) (engine.SetConfigurationResult, error) + statsFunc func(context.Context, engine.StatsRequest) (engine.StatsResult, error) + inspectFunc func(context.Context, engine.InspectRequest) (engine.InspectResult, error) + verifyFunc func(context.Context, engine.VerifyRequest) (engine.VerifyResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -131,6 +134,27 @@ func (s stubCommandEngine) SetConfiguration(ctx context.Context, req engine.SetC return engine.SetConfigurationResult{}, errors.New("unexpected SetConfiguration call") } +func (s stubCommandEngine) Stats(ctx context.Context, req engine.StatsRequest) (engine.StatsResult, error) { + if s.statsFunc != nil { + return s.statsFunc(ctx, req) + } + return engine.StatsResult{}, errors.New("unexpected Stats call") +} + +func (s stubCommandEngine) Inspect(ctx context.Context, req engine.InspectRequest) (engine.InspectResult, error) { + if s.inspectFunc != nil { + return s.inspectFunc(ctx, req) + } + return engine.InspectResult{}, errors.New("unexpected Inspect call") +} + +func (s stubCommandEngine) Verify(ctx context.Context, req engine.VerifyRequest) (engine.VerifyResult, error) { + if s.verifyFunc != nil { + return s.verifyFunc(ctx, req) + } + return engine.VerifyResult{}, errors.New("unexpected Verify call") +} + func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 35d89355..4ce96d90 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -93,11 +93,13 @@ ## Phase 14 — Inspect, Stats, and Verify Routing -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 14 inspect, stats, and verify routing](v1.13.12-phase14-inspect-stats-and-verify-routing.md). ## Phase 15 — Repair Activation -**Status:** Not started +**Status:** Next ## Phase 16 — Recovery Activation diff --git a/docs/release/v1.13/v1.13.12-phase14-inspect-stats-and-verify-routing.md b/docs/release/v1.13/v1.13.12-phase14-inspect-stats-and-verify-routing.md new file mode 100644 index 00000000..6ea93534 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase14-inspect-stats-and-verify-routing.md @@ -0,0 +1,58 @@ +# Coldkeep v1.13.12 Phase 14 — Inspect, Stats, and Verify Routing + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `14 — Inspect, Stats, and Verify Routing` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Production ownership + +Production stats and inspect now construct an injected engine, issue one typed +operation, replay the engine's ordered neutral trace events when requested, and +project the neutral result into the established renderer types. Production +inspect no longer constructs or calls an observability service. + +Production verify now issues one `Engine.Verify` request. Execution and the +complete verification summary are owned by that operation; the CLI no longer +runs a second direct database summary query. The CLI retains argument parsing, +public error compatibility, JSON/text rendering, and the operational hint. + +Inspect summary, metadata, relation metadata, warnings, and traces cross the +boundary as engine-owned DTOs. Recursive tagged values preserve exact integer +tokens during compatibility projection. Typed engine `not_found` errors retain +the existing public `NOT_FOUND` inspect behavior. + +Direct observability service construction remains only in explicitly excluded +simulation/benchmark tooling over isolated simulated repositories. Legacy +inspect and verify fixture seams are test-only. + +## Compatibility + +Existing stats and inspect human/JSON shapes, trace ordering, numeric fidelity, +inspect validation and labels, verify levels, verification failure mapping, +summary counters, file IDs, hints, and exit codes remain unchanged. No schema, +migration, repository format, storage semantics, backend default, transaction, +lock, or coordination behavior changed. + +## Validation + +- Direct CLI one-operation routing and exact-value projection suites: pass. +- Existing stats, inspect, trace, verify, JSON, text, and error suites: pass. +- Shared SQLite/PostgreSQL stats, inspect, and verify contracts: pass. +- Focused CLI, engine, observability, and verify race suites: pass. +- Full `go test -count=1 ./...`: pass with an isolated module cache populated + from the local read-only module proxy. +- Validation root: `/tmp/coldkeep-v11312-phase14.8obDV7/`. + +PostgreSQL validation created and dropped exactly: + +- `coldkeep_bt_testenginereadstatsandinspectacrossbackend_2e51f9_1` +- `coldkeep_bt_testenginereadverifyacrossbackends_postgre_2e51f9_2` + +A read-only maintenance query confirmed both exact names are absent. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 15 is authorized to activate engine-owned repair validation, planning, +execution, invariant classification, fail-fast behavior, and aggregate results. From 9159591b8317659e68af9a4f5e3bd197560e8b37 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 15:09:14 +0200 Subject: [PATCH 16/27] Activate engine-owned repair orchestration --- cmd/coldkeep/legacy_repair_seams_test.go | 106 +++++++++ cmd/coldkeep/main.go | 210 ++++++------------ cmd/coldkeep/repair_engine_routing_test.go | 89 ++++++++ cmd/coldkeep/stored_path_engine_adapters.go | 6 + .../stored_path_engine_routing_test.go | 8 + docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- .../v1.13.12-phase15-repair-activation.md | 80 +++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 35 ++- internal/engine/candidates.go | 48 ++-- internal/engine/candidates_test.go | 10 +- internal/engine/contracts_test.go | 29 +-- internal/engine/engine.go | 5 + internal/engine/repair.go | 137 ++++++++++++ .../engine/repair_backend_contract_test.go | 172 ++++++++++++++ .../maintenance/repair_chunk_refcounts.go | 15 +- internal/maintenance/repair_refcounts.go | 15 +- 18 files changed, 761 insertions(+), 218 deletions(-) create mode 100644 cmd/coldkeep/legacy_repair_seams_test.go create mode 100644 cmd/coldkeep/repair_engine_routing_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase15-repair-activation.md create mode 100644 internal/engine/repair.go create mode 100644 internal/engine/repair_backend_contract_test.go diff --git a/cmd/coldkeep/legacy_repair_seams_test.go b/cmd/coldkeep/legacy_repair_seams_test.go new file mode 100644 index 00000000..9aebc12b --- /dev/null +++ b/cmd/coldkeep/legacy_repair_seams_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/invariants" + "github.com/franchoy/coldkeep/internal/maintenance" +) + +var repairLogicalRefCountsPhase = maintenance.RepairLogicalRefCountsResultRun +var repairChunkLiveRefCountsPhase = maintenance.RepairChunkLiveRefCountsResultRun +var productionRepairCommandEngine = newRepairCommandEngine +var productionRepairDBConnector = connectRepairDBPhase + +func init() { + connectRepairDBPhase = func() (*sql.DB, error) { return sql.Open("sqlite3", ":memory:") } + newRepairCommandEngine = func(*sql.DB) (engine.Engine, error) { + return legacyRepairTestEngine{}, nil + } +} + +type legacyRepairTestEngine struct{ engine.Engine } + +func (legacyRepairTestEngine) Repair(ctx context.Context, req engine.RepairRequest) (engine.RepairResult, error) { + result := engine.RepairResult{Targets: make([]engine.RepairTargetResult, 0, len(req.Targets))} + seen := make(map[engine.RepairTarget]struct{}, len(req.Targets)) + for _, raw := range req.Targets { + if err := ctx.Err(); err != nil { + return result, err + } + normalized := strings.TrimSpace(raw) + item := engine.RepairTargetResult{RawTarget: normalized, Target: engine.RepairTarget(normalized)} + if normalized == "" { + item.RawTarget = raw + item.Target = "" + item.Status = engine.BatchItemFailed + item.Message = fmt.Sprintf("invalid repair target %q", raw) + appendLegacyRepairResult(&result, item) + continue + } + if item.Target != engine.RepairTargetRefCounts && item.Target != engine.RepairTargetChunkLiveRefCounts { + item.RawTarget = raw + item.Target = "" + item.Status = engine.BatchItemFailed + item.Message = fmt.Sprintf("unknown repair target %q", raw) + appendLegacyRepairResult(&result, item) + continue + } + if _, duplicate := seen[item.Target]; duplicate { + item.Status = engine.BatchItemSkipped + item.Message = "duplicate target" + appendLegacyRepairResult(&result, item) + continue + } + seen[item.Target] = struct{}{} + + var err error + switch item.Target { + case engine.RepairTargetRefCounts: + var value maintenance.RepairLogicalRefCountsResult + value, err = repairLogicalRefCountsPhase() + if err == nil { + item.ScannedRows, item.UpdatedRows, item.OrphanRows = value.ScannedLogicalFiles, value.UpdatedLogicalFiles, value.OrphanPhysicalFileRows + item.Message = fmt.Sprintf("repaired scanned_logical_files=%d updated_logical_files=%d orphan_physical_file_rows=%d", value.ScannedLogicalFiles, value.UpdatedLogicalFiles, value.OrphanPhysicalFileRows) + } + case engine.RepairTargetChunkLiveRefCounts: + var value maintenance.RepairChunkLiveRefCountsResult + value, err = repairChunkLiveRefCountsPhase() + if err == nil { + item.ScannedRows, item.UpdatedRows = value.ScannedChunks, value.UpdatedChunks + item.Message = fmt.Sprintf("repaired scanned_chunks=%d updated_chunks=%d", value.ScannedChunks, value.UpdatedChunks) + } + } + if err == nil { + item.Status = engine.BatchItemOK + } else { + item.Status = engine.BatchItemFailed + item.Message = fmt.Sprintf("repair %s failed: %v", item.Target, err) + if code, ok := invariants.Code(err); ok { + item.InvariantCode = code + item.RecommendedAction = invariants.RecommendedActionForCode(code) + } + } + appendLegacyRepairResult(&result, item) + if req.FailFast && item.Status == engine.BatchItemFailed { + break + } + } + return result, nil +} + +func appendLegacyRepairResult(result *engine.RepairResult, item engine.RepairTargetResult) { + result.Targets = append(result.Targets, item) + switch item.Status { + case engine.BatchItemOK: + result.Summary.OK++ + case engine.BatchItemFailed: + result.Summary.Failed++ + case engine.BatchItemSkipped: + result.Summary.Skipped++ + } +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 3c8e9f3a..d810f080 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -196,17 +196,6 @@ var doctorSchemaVersionPhase = db.QueryCurrentSchemaVersion var doctorVerifyPhase = maintenance.VerifyCommandWithContainersDir var doctorSystemAuditPhase = maintenance.CollectSystemAuditSummary -// Current intentional CLI/domain ownership: repair remains direct maintenance -// execution rather than active engine ownership. Phase 14 made this boundary -// decision, and the Phase 16 honesty proof confirmed it. Any future activation -// belongs to an explicit early-v2.0 design; this direct hook is intentional. -var repairLogicalRefCountsPhase = maintenance.RepairLogicalRefCountsResultRun - -// Current intentional CLI/domain ownership: chunk live-ref-count repair remains -// a direct maintenance hook, not an engine-routed workflow. Phase 14 made this -// boundary decision, and the Phase 16 honesty proof confirmed it. Any future -// early-v2.0 activation design must be explicit and behavior-preserving. -var repairChunkLiveRefCountsPhase = maintenance.RepairChunkLiveRefCountsResultRun var storeByFilePhase = func(sgctx *storage.StorageContext, path, codecName string) (storage.StoreFileResult, error) { if sgctx == nil || sgctx.DB == nil { return storage.StoreFileResult{}, fmt.Errorf("store: storage context DB is required") @@ -2303,12 +2292,12 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return usageErrorf("Usage: coldkeep repair ref-counts --batch [--input ] [--fail-fast] [--output ]") } - prepared := prepareRepairTargets(rawTargets) - if len(prepared) == 0 { - return usageErrorf("no valid repair targets after parsing input") + failFast := parsed.hasFlag("fail-fast", "failFast") + result, err := executeRepairEngine(rawRepairTargetValues(rawTargets), failFast) + if err != nil { + return err } - - report := executeRepairPrepared(parsed.hasFlag("fail-fast", "failFast"), prepared) + report := repairBatchReport(result, failFast) return emitBatchCommandReport("repair", report, outputMode) } @@ -2319,9 +2308,9 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error target := strings.TrimSpace(parsed.positionals[0]) switch target { case "ref-counts": - result, err := repairLogicalRefCountsPhase() + result, err := executeSingleRepairEngine(target) if err != nil { - return verifyError(fmt.Errorf("repair ref-counts failed: %w", err)) + return err } if outputMode == outputModeJSON { @@ -2330,9 +2319,9 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error "command": "repair", "data": map[string]any{ "target": "ref-counts", - "scanned_logical_files": result.ScannedLogicalFiles, - "updated_logical_files": result.UpdatedLogicalFiles, - "orphan_physical_file_rows": result.OrphanPhysicalFileRows, + "scanned_logical_files": result.ScannedRows, + "updated_logical_files": result.UpdatedRows, + "orphan_physical_file_rows": result.OrphanRows, }, } encoded, _ := json.Marshal(payload) @@ -2341,17 +2330,17 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } fmt.Printf("Recomputed logical_file.ref_count from physical_file rows. scanned_logical_files=%d updated_logical_files=%d orphan_physical_file_rows=%d\n", - result.ScannedLogicalFiles, - result.UpdatedLogicalFiles, - result.OrphanPhysicalFileRows, + result.ScannedRows, + result.UpdatedRows, + result.OrphanRows, ) fmt.Printf("Hint: %s\n", doctorOperationalHint) return nil case "chunk-live-ref-counts": - result, err := repairChunkLiveRefCountsPhase() + result, err := executeSingleRepairEngine(target) if err != nil { - return verifyError(fmt.Errorf("repair chunk-live-ref-counts failed: %w", err)) + return err } if outputMode == outputModeJSON { @@ -2360,8 +2349,8 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error "command": "repair", "data": map[string]any{ "target": "chunk-live-ref-counts", - "scanned_chunks": result.ScannedChunks, - "updated_chunks": result.UpdatedChunks, + "scanned_chunks": result.ScannedRows, + "updated_chunks": result.UpdatedRows, }, } encoded, _ := json.Marshal(payload) @@ -2370,8 +2359,8 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } fmt.Printf("Recomputed chunk.live_ref_count from file_chunk rows. scanned_chunks=%d updated_chunks=%d\n", - result.ScannedChunks, - result.UpdatedChunks, + result.ScannedRows, + result.UpdatedRows, ) fmt.Printf("Hint: %s\n", doctorOperationalHint) return nil @@ -2380,129 +2369,64 @@ func runRepairCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } } -type preparedRepairTarget struct { - Target string - Executable bool - Result batch.ItemResult +func rawRepairTargetValues(raw []batch.RawTarget) []string { + values := make([]string, len(raw)) + for i, item := range raw { + values[i] = item.Value + } + return values } -func prepareRepairTargets(raw []batch.RawTarget) []preparedRepairTarget { - prepared := make([]preparedRepairTarget, 0, len(raw)) - seen := make(map[string]struct{}, len(raw)) - - for _, item := range raw { - target := strings.TrimSpace(item.Value) - if target == "" { - prepared = append(prepared, preparedRepairTarget{ - Executable: false, - Result: batch.ItemResult{ - RawValue: item.Value, - Status: batch.ResultFailed, - Message: fmt.Sprintf("invalid repair target %q", item.Value), - }, - }) - continue - } - - if target != "ref-counts" && target != "chunk-live-ref-counts" { - prepared = append(prepared, preparedRepairTarget{ - Executable: false, - Result: batch.ItemResult{ - RawValue: item.Value, - Status: batch.ResultFailed, - Message: fmt.Sprintf("unknown repair target %q", item.Value), - }, - }) - continue - } +func executeRepairEngine(targets []string, failFast bool) (engine.RepairResult, error) { + dbconn, err := connectRepairDBPhase() + if err != nil { + return engine.RepairResult{}, fmt.Errorf("connect repair database: %w", err) + } + defer func() { _ = dbconn.Close() }() + eng, err := newRepairCommandEngine(dbconn) + if err != nil { + return engine.RepairResult{}, err + } + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return eng.Repair(ctx, engine.RepairRequest{Targets: append([]string(nil), targets...), FailFast: failFast}) +} - if _, exists := seen[target]; exists { - prepared = append(prepared, preparedRepairTarget{ - Executable: false, - Result: batch.ItemResult{ - RawValue: target, - Status: batch.ResultSkipped, - Message: "duplicate target", - }, - }) - continue +func executeSingleRepairEngine(target string) (engine.RepairTargetResult, error) { + result, err := executeRepairEngine([]string{target}, false) + if err != nil { + return engine.RepairTargetResult{}, err + } + if len(result.Targets) != 1 { + return engine.RepairTargetResult{}, fmt.Errorf("repair engine returned %d target results, want 1", len(result.Targets)) + } + item := result.Targets[0] + if item.Status == engine.BatchItemFailed { + failure := error(fmt.Errorf("%s", item.Message)) + if item.InvariantCode != "" { + failure = invariants.New(item.InvariantCode, item.Message, nil) } - - seen[target] = struct{}{} - prepared = append(prepared, preparedRepairTarget{Target: target, Executable: true}) + return engine.RepairTargetResult{}, verifyError(failure) } - - return prepared + return item, nil } -func executeRepairPrepared(failFast bool, targets []preparedRepairTarget) batch.Report { - results := make([]batch.ItemResult, 0, len(targets)) - - for _, target := range targets { - if !target.Executable { - results = append(results, target.Result) - continue +func repairBatchReport(result engine.RepairResult, failFast bool) batch.Report { + items := make([]batch.ItemResult, len(result.Targets)) + for i, item := range result.Targets { + status := batch.ResultFailed + switch item.Status { + case engine.BatchItemOK: + status = batch.ResultSuccess + case engine.BatchItemSkipped: + status = batch.ResultSkipped } - - switch target.Target { - case "ref-counts": - result, err := repairLogicalRefCountsPhase() - if err != nil { - item := batch.ItemResult{ - RawValue: target.Target, - Status: batch.ResultFailed, - Message: fmt.Sprintf("repair ref-counts failed: %v", err), - } - if code, ok := invariants.Code(err); ok { - item.InvariantCode = code - item.RecommendedAction = invariants.RecommendedActionForCode(code) - } - results = append(results, item) - if failFast { - break - } - continue - } - - results = append(results, batch.ItemResult{ - RawValue: target.Target, - Status: batch.ResultSuccess, - Message: fmt.Sprintf( - "repaired scanned_logical_files=%d updated_logical_files=%d orphan_physical_file_rows=%d", - result.ScannedLogicalFiles, - result.UpdatedLogicalFiles, - result.OrphanPhysicalFileRows, - ), - }) - - case "chunk-live-ref-counts": - result, err := repairChunkLiveRefCountsPhase() - if err != nil { - item := batch.ItemResult{ - RawValue: target.Target, - Status: batch.ResultFailed, - Message: fmt.Sprintf("repair chunk-live-ref-counts failed: %v", err), - } - if code, ok := invariants.Code(err); ok { - item.InvariantCode = code - item.RecommendedAction = invariants.RecommendedActionForCode(code) - } - results = append(results, item) - if failFast { - break - } - continue - } - - results = append(results, batch.ItemResult{ - RawValue: target.Target, - Status: batch.ResultSuccess, - Message: fmt.Sprintf("repaired scanned_chunks=%d updated_chunks=%d", result.ScannedChunks, result.UpdatedChunks), - }) + items[i] = batch.ItemResult{ + RawValue: item.RawTarget, Status: status, Message: item.Message, + InvariantCode: item.InvariantCode, RecommendedAction: item.RecommendedAction, } } - - report := batch.NewReport(batch.OperationRepair, false, results) + report := batch.NewReport(batch.OperationRepair, false, items) report.ExecutionMode = batch.ExecutionModeContinueOnError if failFast { report.ExecutionMode = batch.ExecutionModeFailFast diff --git a/cmd/coldkeep/repair_engine_routing_test.go b/cmd/coldkeep/repair_engine_routing_test.go new file mode 100644 index 00000000..dc92ea99 --- /dev/null +++ b/cmd/coldkeep/repair_engine_routing_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "database/sql" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/invariants" +) + +func TestRunRepairBatchUsesEngineAndPreservesProjection(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + var calls int + installRepairCommandStubs(t, dbconn, stubCommandEngine{repairFunc: func(_ context.Context, req engine.RepairRequest) (engine.RepairResult, error) { + calls++ + if !req.FailFast || len(req.Targets) != 3 || req.Targets[0] != " ref-counts " || req.Targets[1] != "unknown" || req.Targets[2] != "ref-counts" { + t.Fatalf("unexpected Repair request: %+v", req) + } + return engine.RepairResult{Targets: []engine.RepairTargetResult{ + {RawTarget: "ref-counts", Target: engine.RepairTargetRefCounts, Status: engine.BatchItemOK, Message: "repaired scanned_logical_files=1 updated_logical_files=1 orphan_physical_file_rows=0"}, + {RawTarget: "unknown", Status: engine.BatchItemFailed, Message: `unknown repair target "unknown"`}, + {RawTarget: "ref-counts", Target: engine.RepairTargetRefCounts, Status: engine.BatchItemSkipped, Message: "duplicate target"}, + }}, nil + }}) + + output := captureStdout(t, func() { + if err := runRepairCommand(parsedCommandLine{method: "repair", positionals: []string{" ref-counts ", "unknown", "ref-counts"}, flags: map[string][]string{"batch": {"true"}, "fail-fast": {"true"}}}, outputModeJSON); err == nil { + t.Fatal("mixed batch should retain its failed exit result") + } + }) + if calls != 1 { + t.Fatalf("Repair calls=%d want 1", calls) + } + for _, want := range []string{`"command":"repair"`, `"execution_mode":"fail_fast"`, `"status":"failed"`, `"status":"skipped"`, `"message":"duplicate target"`} { + if !strings.Contains(output, want) { + t.Fatalf("batch JSON missing %q: %s", want, output) + } + } +} + +func TestRunRepairSingleUsesEngineCompatibilityProjection(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installRepairCommandStubs(t, dbconn, stubCommandEngine{repairFunc: func(_ context.Context, req engine.RepairRequest) (engine.RepairResult, error) { + if req.FailFast || len(req.Targets) != 1 || req.Targets[0] != "ref-counts" { + t.Fatalf("unexpected Repair request: %+v", req) + } + return engine.RepairResult{Targets: []engine.RepairTargetResult{{RawTarget: "ref-counts", Target: engine.RepairTargetRefCounts, ScannedRows: 4, UpdatedRows: 2, OrphanRows: 0, Status: engine.BatchItemOK}}}, nil + }}) + output := captureStdout(t, func() { + if err := runRepairCommand(parsedCommandLine{method: "repair", positionals: []string{"ref-counts"}}, outputModeJSON); err != nil { + t.Fatalf("runRepairCommand: %v", err) + } + }) + if strings.TrimSpace(output) != `{"command":"repair","data":{"orphan_physical_file_rows":0,"scanned_logical_files":4,"target":"ref-counts","updated_logical_files":2},"status":"ok"}` { + t.Fatalf("unexpected JSON projection: %s", output) + } +} + +func TestRunRepairSinglePreservesInvariantExitClassification(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installRepairCommandStubs(t, dbconn, stubCommandEngine{repairFunc: func(context.Context, engine.RepairRequest) (engine.RepairResult, error) { + return engine.RepairResult{Targets: []engine.RepairTargetResult{{ + RawTarget: "ref-counts", Target: engine.RepairTargetRefCounts, Status: engine.BatchItemFailed, + Message: "repair refused", InvariantCode: invariants.CodeRepairRefusedOrphanRows, + }}}, nil + }}) + err := runRepairCommand(parsedCommandLine{method: "repair", positionals: []string{"ref-counts"}}, outputModeText) + if err == nil || classifyExitCode(err) != exitVerify || !strings.Contains(err.Error(), "repair refused") { + t.Fatalf("expected verify-class invariant error, got %v (exit=%d)", err, classifyExitCode(err)) + } +} + +func installRepairCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { + t.Helper() + originalConnect := connectRepairDBPhase + originalNewEngine := newRepairCommandEngine + t.Cleanup(func() { + connectRepairDBPhase = originalConnect + newRepairCommandEngine = originalNewEngine + }) + connectRepairDBPhase = func() (*sql.DB, error) { + return dbconn, nil + } + newRepairCommandEngine = func(*sql.DB) (engine.Engine, error) { + return stub, nil + } +} diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index 77ecaa33..cb8c2358 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -52,6 +52,12 @@ var newObservabilityCommandEngine = func(sgctx storage.StorageContext) (engine.E return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) } +var newRepairCommandEngine = func(dbconn *sql.DB) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn}) +} + +var connectRepairDBPhase = db.ConnectDB + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 25c97f1a..2b0d17df 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -34,6 +34,7 @@ type stubCommandEngine struct { statsFunc func(context.Context, engine.StatsRequest) (engine.StatsResult, error) inspectFunc func(context.Context, engine.InspectRequest) (engine.InspectResult, error) verifyFunc func(context.Context, engine.VerifyRequest) (engine.VerifyResult, error) + repairFunc func(context.Context, engine.RepairRequest) (engine.RepairResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -155,6 +156,13 @@ func (s stubCommandEngine) Verify(ctx context.Context, req engine.VerifyRequest) return engine.VerifyResult{}, errors.New("unexpected Verify call") } +func (s stubCommandEngine) Repair(ctx context.Context, req engine.RepairRequest) (engine.RepairResult, error) { + if s.repairFunc != nil { + return s.repairFunc(ctx, req) + } + return engine.RepairResult{}, errors.New("unexpected Repair call") +} + func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index c63d95f4..83397b59 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–7 are complete; Phase 8 is next. + 0–15 are complete; Phase 16 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 4ce96d90..b980abba 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -99,11 +99,13 @@ ## Phase 15 — Repair Activation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 15 repair activation](v1.13.12-phase15-repair-activation.md). ## Phase 16 — Recovery Activation -**Status:** Not started +**Status:** Next ## Phase 17 — Doctor Activation diff --git a/docs/release/v1.13/v1.13.12-phase15-repair-activation.md b/docs/release/v1.13/v1.13.12-phase15-repair-activation.md new file mode 100644 index 00000000..5e4696c0 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase15-repair-activation.md @@ -0,0 +1,80 @@ +# Coldkeep v1.13.12 Phase 15 — Repair Activation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `15 — Repair Activation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Production ownership + +`Engine.Repair` is now the active repair boundary. It accepts ordered raw +targets and owns whitespace normalization, validation, deterministic +deduplication, duplicate-skipped results, target execution, fail-fast behavior, +invariant classification, and aggregate counts. The provisional dry-run, +input-file, limit, batch-switch, and separate single-target contract fields are +removed. + +The two repair implementations accept the engine operation context and retain +their independent transaction boundaries. Logical ref-count repair still +refuses orphan `physical_file` rows before mutation; chunk live-ref-count +repair still derives truth from `file_chunk`. Both retain exact row-cardinality +checks and roll back on failure. + +Production CLI repair now opens only the configured database, constructs an +injected engine, and issues one request for either the single or batch form. +The CLI retains argument/file ingestion, established batch-report projection, +specialized single-target text/JSON rendering, hints, and exit compatibility. +Legacy direct-maintenance seams exist only in tests. + +## Compatibility + +Command syntax, input-file semantics, raw-target reporting, validation text, +duplicate order, fail-fast order, counters, batch JSON, single-target JSON/text, +invariant metadata, exit codes, transactions, schema, repository format, +storage bytes, backend selection, locking, and coordination are unchanged. +Repair does not construct or finalize a storage writer. + +## Validation + +- Direct engine execution, validation-ordering, typed argument/cancellation, + fail-fast, rollback, and invariant suites: pass on SQLite and PostgreSQL. +- Direct CLI one-operation routing and compatibility projection suites: pass. +- Existing engine, maintenance, and CLI suites: pass. +- Focused engine, maintenance, and CLI race suites: pass. +- Full `go test -count=1 ./...`: pass. +- Release-state validator: development state, 1.13.12, zero violations after + tracker reconciliation. +- Validation root: `/tmp/coldkeep-v11312-phase15.1nHeAE/`. + +PostgreSQL validation created and dropped exactly the following scratch +databases. The first four reached schema-bootstrap preflight only; the next +four exposed and then closed a boolean-fixture portability error; the next +four completed the first passing parity run; and the final four completed the +exact-final-worktree parity run after cancellation propagation was hardened. + +- `coldkeep_bt_testenginerepairacrossbackends_postgres_30d3f0_1` +- `coldkeep_bt_testenginerepairvalidationorderingacrossba_30d3f0_2` +- `coldkeep_bt_testenginerepairtypedboundaryerrors_postgr_30d3f0_3` +- `coldkeep_bt_testenginerepairfailfastoninvariantviolati_30d3f0_4` +- `coldkeep_bt_testenginerepairacrossbackends_postgres_30d694_1` +- `coldkeep_bt_testenginerepairvalidationorderingacrossba_30d694_2` +- `coldkeep_bt_testenginerepairtypedboundaryerrors_postgr_30d694_3` +- `coldkeep_bt_testenginerepairfailfastoninvariantviolati_30d694_4` +- `coldkeep_bt_testenginerepairacrossbackends_postgres_30d975_1` +- `coldkeep_bt_testenginerepairvalidationorderingacrossba_30d975_2` +- `coldkeep_bt_testenginerepairtypedboundaryerrors_postgr_30d975_3` +- `coldkeep_bt_testenginerepairfailfastoninvariantviolati_30d975_4` +- `coldkeep_bt_testenginerepairacrossbackends_postgres_326e73_1` +- `coldkeep_bt_testenginerepairvalidationorderingacrossba_326e73_2` +- `coldkeep_bt_testenginerepairtypedboundaryerrors_postgr_326e73_3` +- `coldkeep_bt_testenginerepairfailfastoninvariantviolati_326e73_4` + +A read-only maintenance query over all sixteen exact names returned no rows. +No ordinary database was reset, migrated, truncated, or dropped. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 16 is authorized to activate DB-injected engine recovery and route both +explicit and startup recovery through the same boundary while preserving +journal handling, counters, idempotence, and startup ordering. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 13044dfe..1dedd7bd 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–12 are Complete. Phase 13 is Next: complete snapshot list, tree, show, -stats, and every diff mode behind typed engine read operations with repeated -selectors and frozen filtering/output semantics. +Phases 0–15 are Complete. Phase 16 is Next: activate one DB-injected recovery +engine operation and route explicit and startup recovery through it without +changing recovery ordering, counters, journal handling, or idempotence. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index be52a61c..dd4d8762 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -204,19 +204,46 @@ ## Phase 13 — Snapshot Read-Side Completion -**Phase status:** Next +**Phase status:** Complete + +- [x] Snapshot list, tree, show, stats, and every diff mode are single typed + engine operations with neutral DTOs and repeated selectors. +- [x] Production tree planning consumes the catalog graph and no snapshot read + command retains a mixed direct-domain route. +- [x] Filtering, ordering, JSON/text compatibility, dual-backend parity, full + regression, and race coverage pass. ## Phase 14 — Inspect, Stats, and Verify Routing -**Phase status:** Not started +**Phase status:** Complete + +- [x] Production inspect, stats, and verify each route through one engine + operation; direct observability/verification execution is test-only or + explicitly excluded isolated tooling. +- [x] Neutral inspect values preserve exact numbers, ordered traces, existing + rendering, diagnostics, summaries, and error/exit behavior. +- [x] Focused routing, dual-backend, full regression, and race suites pass. ## Phase 15 — Repair Activation -**Phase status:** Not started +**Phase status:** Complete + +- [x] `Engine.Repair` owns target validation, whitespace normalization, + deterministic deduplication, duplicate-skipped reporting, ordered execution, + fail-fast behavior, invariant classification, and aggregate results. +- [x] The active repair request/result graph is neutral and contains no + provisional dry-run, input-path, limit, or single-target facade fields. +- [x] Production CLI repair performs file ingestion and compatibility + projection only; both single and batch forms call one engine operation. +- [x] Logical and chunk recomputations retain independent transactions, + fail-closed orphan behavior, rollback, counters, messages, JSON/text shape, + and exit classification. +- [x] Direct engine, CLI routing, maintenance, race, full regression, and shared + SQLite/PostgreSQL contract suites pass. ## Phase 16 — Recovery Activation -**Phase status:** Not started +**Phase status:** Next ## Phase 17 — Doctor Activation diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index a4f76658..f300349f 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -827,51 +827,33 @@ const ( RepairTargetChunkLiveRefCounts RepairTarget = "chunk-live-ref-counts" ) -// RepairRequest is a candidate-only request contract for a future Repair -// operation. Repair is not a method on the current Engine interface. -// -// Request/result presence must not be mistaken for active engine ownership. -// Phase 14 and the Phase 16 honesty proof confirmed current CLI/domain -// ownership. Any early-v2.0 activation design must be explicit and -// behavior-preserving. +// RepairRequest is the active ordered repair contract. Targets contain raw +// caller values so validation, normalization, duplicate detection, and +// deterministic reporting remain engine-owned. Input-file ingestion remains a +// caller responsibility. type RepairRequest struct { - // Target selects the single-target repair (when Batch is false). - Target RepairTarget - // Batch processes multiple targets. - Batch bool - // Targets is the explicit batch target list. - Targets []RepairTarget + Targets []string // FailFast stops a batch on the first failure. FailFast bool - // InputPath is an optional batch-input source. - // - // Batch-input parsing remains caller-side under current ownership and would - // require an explicit decision if Repair were activated. - InputPath string - // DryRun simulates without mutating, where supported. - DryRun bool - // Limit caps the number of rows processed when greater than zero. - Limit int } // RepairTargetResult is the outcome of a single repair target. type RepairTargetResult struct { - Target RepairTarget + RawTarget string + Target RepairTarget // ScannedRows and UpdatedRows are generic counters covering both // logical-file and chunk recomputations. - ScannedRows int - UpdatedRows int + ScannedRows int64 + UpdatedRows int64 // OrphanRows captures orphan physical-file rows for ref-count repair. - OrphanRows int - Status BatchItemStatus - Error string + OrphanRows int64 + Status BatchItemStatus + Message string + InvariantCode string + RecommendedAction string } -// RepairResult is a candidate-only result contract for a future Repair -// operation. Repair is not a method on the current Engine interface. -// Phase 14 and the Phase 16 honesty proof confirmed current CLI/domain -// ownership; any early-v2.0 activation design must be explicit and -// behavior-preserving. +// RepairResult is the complete aggregate result of the processed targets. type RepairResult struct { // Targets holds per-target outcomes. Targets []RepairTargetResult diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index 008da7e5..1da83cb0 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) @@ -52,7 +52,6 @@ func TestEngineActiveInterfaceExcludesStillInactiveOperations(t *testing.T) { } for _, forbidden := range []string{ - "Repair", "SnapshotRepair", "RepairPlan", "Recover", @@ -101,13 +100,6 @@ type candidateOnlyOperationCase struct { func candidateOnlyOperationCases() []candidateOnlyOperationCase { return []candidateOnlyOperationCase{ - { - name: "repair", - requestType: engine.RepairRequest{}, - resultType: engine.RepairResult{}, - forbiddenMethod: "Repair", - futureDisposition: "early v2.0 activation-design decision", - }, { name: "recover", requestType: engine.RecoverRequest{}, diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 64979f22..b0628c48 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -68,6 +68,9 @@ func activeCoreContractTypes() []struct { {"GarbageCollectResult", engine.GarbageCollectResult{}}, {"VerifyRequest", engine.VerifyRequest{}}, {"VerifyResult", engine.VerifyResult{}}, + {"RepairRequest", engine.RepairRequest{}}, + {"RepairTargetResult", engine.RepairTargetResult{}}, + {"RepairResult", engine.RepairResult{}}, {"SnapshotMeta", engine.SnapshotMeta{}}, {"SnapshotListRequest", engine.SnapshotListRequest{}}, {"SnapshotListResult", engine.SnapshotListResult{}}, @@ -136,9 +139,6 @@ func candidateCorrectiveContractTypes() []struct { name string val any }{ - {"RepairRequest", engine.RepairRequest{}}, - {"RepairTargetResult", engine.RepairTargetResult{}}, - {"RepairResult", engine.RepairResult{}}, {"RecoverRequest", engine.RecoverRequest{}}, {"RecoverResult", engine.RecoverResult{}}, } @@ -603,27 +603,22 @@ func assertSnapshotRestoreContract(t *testing.T, after time.Time, before time.Ti // TestRepairContractRepresentsTargetsAndBatch proves repair can express the // single targets and batch/fail-fast behavior. func TestRepairContractRepresentsTargetsAndBatch(t *testing.T) { - single := engine.RepairRequest{Target: engine.RepairTargetRefCounts} - if single.Target != engine.RepairTargetRefCounts { - t.Error("single ref-counts repair not representable") - } - batch := engine.RepairRequest{ - Batch: true, - Targets: []engine.RepairTarget{engine.RepairTargetRefCounts, engine.RepairTargetChunkLiveRefCounts}, - FailFast: true, - InputPath: "/in", + request := engine.RepairRequest{ + Targets: []string{" ref-counts ", "chunk-live-ref-counts", "ref-counts"}, + FailFast: true, } - if !batch.Batch || len(batch.Targets) != 2 { + if !request.FailFast || len(request.Targets) != 3 { t.Error("batch repair not representable") } res := engine.RepairResult{ Targets: []engine.RepairTargetResult{ - {Target: engine.RepairTargetRefCounts, ScannedRows: 10, UpdatedRows: 2, OrphanRows: 1, Status: engine.BatchItemOK}, - {Target: engine.RepairTargetChunkLiveRefCounts, ScannedRows: 5, UpdatedRows: 0, Status: engine.BatchItemOK}, + {RawTarget: "ref-counts", Target: engine.RepairTargetRefCounts, ScannedRows: 10, UpdatedRows: 2, OrphanRows: 1, Status: engine.BatchItemOK}, + {RawTarget: "chunk-live-ref-counts", Target: engine.RepairTargetChunkLiveRefCounts, ScannedRows: 5, UpdatedRows: 0, Status: engine.BatchItemOK}, + {RawTarget: "ref-counts", Target: engine.RepairTargetRefCounts, Status: engine.BatchItemSkipped, Message: "duplicate target"}, }, - Summary: engine.BatchSummary{OK: 2}, + Summary: engine.BatchSummary{OK: 2, Skipped: 1}, } - if len(res.Targets) != 2 { + if len(res.Targets) != 3 { t.Fatalf("repair result not representable: %+v", res) } } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index f99895e3..96eb70dc 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -96,6 +96,11 @@ type Engine interface { // SetConfiguration validates and persists one repository write default. SetConfiguration(ctx context.Context, req SetConfigurationRequest) (SetConfigurationResult, error) + // Repair validates, normalizes, deduplicates, and executes ordered catalog + // integrity recomputations. Each target remains transactionally independent; + // FailFast stops after the first execution failure. + Repair(ctx context.Context, req RepairRequest) (RepairResult, error) + // Remove removes logical files from the repository by logical file ID. // Safety invariant: Remove must never make valid data unrecoverable. // Method selection owns addressing semantics: this method is by-ID only. diff --git a/internal/engine/repair.go b/internal/engine/repair.go new file mode 100644 index 00000000..6e1e3a21 --- /dev/null +++ b/internal/engine/repair.go @@ -0,0 +1,137 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/franchoy/coldkeep/internal/invariants" + "github.com/franchoy/coldkeep/internal/maintenance" +) + +func (e *DefaultEngine) Repair(ctx context.Context, req RepairRequest) (RepairResult, error) { + if ctx == nil { + ctx = context.Background() + } + result := RepairResult{Targets: make([]RepairTargetResult, 0, len(req.Targets))} + if len(req.Targets) == 0 { + return result, TranslateErrorAs("repair", ErrorInvalidArgument, fmt.Errorf("repair requires at least one target")) + } + if e == nil || e.config.DB == nil { + return result, TranslateError("repair", fmt.Errorf("repair requires injected database")) + } + + seen := make(map[RepairTarget]struct{}, len(req.Targets)) + for _, raw := range req.Targets { + if err := ctx.Err(); err != nil { + finalizeRepairSummary(&result) + return result, TranslateError("repair", err) + } + item, executable := prepareRepairTarget(raw, seen) + if !executable { + appendRepairTarget(&result, item) + continue + } + seen[item.Target] = struct{}{} + item, executionErr := e.executeRepairTarget(ctx, item) + appendRepairTarget(&result, item) + if executionErr != nil && (errors.Is(executionErr, context.Canceled) || errors.Is(executionErr, context.DeadlineExceeded)) { + return result, TranslateError("repair", executionErr) + } + if req.FailFast && item.Status == BatchItemFailed { + break + } + } + return result, nil +} + +func prepareRepairTarget(raw string, seen map[RepairTarget]struct{}) (RepairTargetResult, bool) { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return RepairTargetResult{ + RawTarget: raw, Status: BatchItemFailed, + Message: fmt.Sprintf("invalid repair target %q", raw), + }, false + } + target := RepairTarget(normalized) + if target != RepairTargetRefCounts && target != RepairTargetChunkLiveRefCounts { + return RepairTargetResult{ + RawTarget: raw, Status: BatchItemFailed, + Message: fmt.Sprintf("unknown repair target %q", raw), + }, false + } + if _, duplicate := seen[target]; duplicate { + return RepairTargetResult{ + RawTarget: normalized, Target: target, Status: BatchItemSkipped, + Message: "duplicate target", + }, false + } + return RepairTargetResult{RawTarget: normalized, Target: target}, true +} + +func (e *DefaultEngine) executeRepairTarget(ctx context.Context, item RepairTargetResult) (RepairTargetResult, error) { + var err error + switch item.Target { + case RepairTargetRefCounts: + var outcome maintenance.RepairLogicalRefCountsResult + outcome, err = maintenance.RepairLogicalRefCountsResultWithDBContext(ctx, e.config.DB) + if err == nil { + item.ScannedRows = outcome.ScannedLogicalFiles + item.UpdatedRows = outcome.UpdatedLogicalFiles + item.OrphanRows = outcome.OrphanPhysicalFileRows + item.Message = fmt.Sprintf( + "repaired scanned_logical_files=%d updated_logical_files=%d orphan_physical_file_rows=%d", + outcome.ScannedLogicalFiles, outcome.UpdatedLogicalFiles, outcome.OrphanPhysicalFileRows, + ) + } + case RepairTargetChunkLiveRefCounts: + var outcome maintenance.RepairChunkLiveRefCountsResult + outcome, err = maintenance.RepairChunkLiveRefCountsResultWithDBContext(ctx, e.config.DB) + if err == nil { + item.ScannedRows = outcome.ScannedChunks + item.UpdatedRows = outcome.UpdatedChunks + item.Message = fmt.Sprintf("repaired scanned_chunks=%d updated_chunks=%d", outcome.ScannedChunks, outcome.UpdatedChunks) + } + default: + err = fmt.Errorf("unknown repair target %q", item.Target) + } + if err == nil { + item.Status = BatchItemOK + return item, nil + } + + item.Status = BatchItemFailed + item.Message = fmt.Sprintf("repair %s failed: %v", item.Target, err) + if code, ok := invariants.Code(err); ok { + item.InvariantCode = code + item.RecommendedAction = invariants.RecommendedActionForCode(code) + } + return item, err +} + +func appendRepairTarget(result *RepairResult, item RepairTargetResult) { + result.Targets = append(result.Targets, item) + switch item.Status { + case BatchItemOK: + result.Summary.OK++ + case BatchItemFailed: + result.Summary.Failed++ + case BatchItemSkipped: + result.Summary.Skipped++ + } +} + +func finalizeRepairSummary(result *RepairResult) { + result.Summary = BatchSummary{} + for _, item := range result.Targets { + switch item.Status { + case BatchItemOK: + result.Summary.OK++ + case BatchItemFailed: + result.Summary.Failed++ + case BatchItemSkipped: + result.Summary.Skipped++ + } + } +} diff --git a/internal/engine/repair_backend_contract_test.go b/internal/engine/repair_backend_contract_test.go new file mode 100644 index 00000000..491baa6d --- /dev/null +++ b/internal/engine/repair_backend_contract_test.go @@ -0,0 +1,172 @@ +package engine_test + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + filestate "github.com/franchoy/coldkeep/internal/status" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineRepairAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + logicalID, chunkID := seedRepairMismatches(t, backend) + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + + result, err := eng.Repair(context.Background(), engine.RepairRequest{Targets: []string{ + " ref-counts ", "chunk-live-ref-counts", "ref-counts", + }}) + if err != nil { + t.Fatalf("Repair: %v", err) + } + if got, want := repairStatuses(result.Targets), []engine.BatchItemStatus{engine.BatchItemOK, engine.BatchItemOK, engine.BatchItemSkipped}; !reflect.DeepEqual(got, want) { + t.Fatalf("target statuses=%v want %v; result=%+v", got, want, result) + } + if result.Summary.OK != 2 || result.Summary.Skipped != 1 || result.Summary.Failed != 0 { + t.Fatalf("summary=%+v", result.Summary) + } + if result.Targets[0].RawTarget != "ref-counts" || result.Targets[0].ScannedRows != 1 || result.Targets[0].UpdatedRows != 1 || result.Targets[0].OrphanRows != 0 { + t.Fatalf("logical result=%+v", result.Targets[0]) + } + if result.Targets[1].ScannedRows != 1 || result.Targets[1].UpdatedRows != 1 { + t.Fatalf("chunk result=%+v", result.Targets[1]) + } + var logicalRefs, chunkRefs int64 + if err := backend.DB.QueryRow(`SELECT ref_count FROM logical_file WHERE id = $1`, logicalID).Scan(&logicalRefs); err != nil { + t.Fatalf("read logical ref_count: %v", err) + } + if err := backend.DB.QueryRow(`SELECT live_ref_count FROM chunk WHERE id = $1`, chunkID).Scan(&chunkRefs); err != nil { + t.Fatalf("read chunk live_ref_count: %v", err) + } + if logicalRefs != 1 || chunkRefs != 1 { + t.Fatalf("repaired counts=(%d,%d), want (1,1)", logicalRefs, chunkRefs) + } + }) +} + +func TestEngineRepairValidationOrderingAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + result, err := eng.Repair(context.Background(), engine.RepairRequest{Targets: []string{" ", "unknown", "ref-counts", " ref-counts "}, FailFast: true}) + if err != nil { + t.Fatalf("Repair validation batch: %v", err) + } + if got, want := repairStatuses(result.Targets), []engine.BatchItemStatus{engine.BatchItemFailed, engine.BatchItemFailed, engine.BatchItemOK, engine.BatchItemSkipped}; !reflect.DeepEqual(got, want) { + t.Fatalf("ordered statuses=%v want %v; result=%+v", got, want, result) + } + if result.Summary.Failed != 2 || result.Summary.OK != 1 || result.Summary.Skipped != 1 { + t.Fatalf("summary=%+v", result.Summary) + } + if !strings.Contains(result.Targets[0].Message, "invalid repair target") || !strings.Contains(result.Targets[1].Message, "unknown repair target") { + t.Fatalf("validation messages=%q, %q", result.Targets[0].Message, result.Targets[1].Message) + } + }) +} + +func TestEngineRepairTypedBoundaryErrors(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + if _, err := eng.Repair(context.Background(), engine.RepairRequest{}); !engine.IsCode(err, engine.ErrorInvalidArgument) { + t.Fatalf("empty request error=%v code=%q", err, engine.CodeOf(err)) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := eng.Repair(ctx, engine.RepairRequest{Targets: []string{"ref-counts"}}); !errors.Is(err, context.Canceled) || !engine.IsCode(err, engine.ErrorCancelled) { + t.Fatalf("cancelled repair error=%v code=%q", err, engine.CodeOf(err)) + } + }) +} + +func TestEngineRepairFailFastOnInvariantViolation(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + logicalID, _ := seedRepairMismatches(t, backend) + if backend.Name == "sqlite" { + if _, err := backend.DB.Exec(`PRAGMA foreign_keys = OFF`); err != nil { + t.Fatalf("disable foreign keys: %v", err) + } + } else { + if _, err := backend.DB.Exec(`ALTER TABLE physical_file DISABLE TRIGGER ALL`); err != nil { + t.Fatalf("disable scratch fixture physical_file triggers: %v", err) + } + } + if _, err := backend.DB.Exec(`INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) VALUES ($1, $2, false)`, "/engine/orphan.bin", int64(999999)); err != nil { + t.Fatalf("insert orphan physical file: %v", err) + } + if backend.Name != "sqlite" { + if _, err := backend.DB.Exec(`ALTER TABLE physical_file ENABLE TRIGGER ALL`); err != nil { + t.Fatalf("re-enable scratch fixture physical_file triggers: %v", err) + } + } + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + result, err := eng.Repair(context.Background(), engine.RepairRequest{ + Targets: []string{"ref-counts", "chunk-live-ref-counts"}, FailFast: true, + }) + if err != nil { + t.Fatalf("Repair aggregate: %v", err) + } + if len(result.Targets) != 1 || result.Targets[0].Status != engine.BatchItemFailed || result.Summary.Failed != 1 { + t.Fatalf("fail-fast result=%+v", result) + } + if result.Targets[0].InvariantCode == "" || result.Targets[0].RecommendedAction == "" { + t.Fatalf("missing invariant classification: %+v", result.Targets[0]) + } + var logicalRefs int64 + if err := backend.DB.QueryRow(`SELECT ref_count FROM logical_file WHERE id = $1`, logicalID).Scan(&logicalRefs); err != nil { + t.Fatalf("read logical ref_count: %v", err) + } + if logicalRefs != 9 { + t.Fatalf("failed repair changed logical ref_count to %d, want original 9", logicalRefs) + } + }) +} + +func seedRepairMismatches(t *testing.T, backend backendtest.Backend) (int64, int64) { + t.Helper() + var logicalID int64 + if err := backend.DB.QueryRow( + `INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) + VALUES ($1, $2, $3, $4, $5, 'v1-simple-rolling') RETURNING id`, + "engine-repair.bin", int64(11), strings.Repeat("a", 64), filestate.LogicalFileCompleted, int64(9), + ).Scan(&logicalID); err != nil { + t.Fatalf("insert logical file: %v", err) + } + if _, err := backend.DB.Exec(`INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) VALUES ($1, $2, false)`, "/engine/repair.bin", logicalID); err != nil { + t.Fatalf("insert physical file: %v", err) + } + var chunkID int64 + if err := backend.DB.QueryRow( + `INSERT INTO chunk (chunk_hash, size, status, live_ref_count, pin_count, retry_count, chunker_version) + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, + strings.Repeat("b", 64), int64(11), filestate.ChunkCompleted, int64(7), int64(0), int64(0), "v1-simple-rolling", + ).Scan(&chunkID); err != nil { + t.Fatalf("insert chunk: %v", err) + } + if _, err := backend.DB.Exec(`INSERT INTO file_chunk (logical_file_id, chunk_id, chunk_order) VALUES ($1, $2, 0)`, logicalID, chunkID); err != nil { + t.Fatalf("insert file chunk: %v", err) + } + return logicalID, chunkID +} + +func repairStatuses(items []engine.RepairTargetResult) []engine.BatchItemStatus { + statuses := make([]engine.BatchItemStatus, len(items)) + for i := range items { + statuses[i] = items[i].Status + } + return statuses +} diff --git a/internal/maintenance/repair_chunk_refcounts.go b/internal/maintenance/repair_chunk_refcounts.go index 8fbfb3bb..112b9125 100644 --- a/internal/maintenance/repair_chunk_refcounts.go +++ b/internal/maintenance/repair_chunk_refcounts.go @@ -19,12 +19,21 @@ type RepairChunkLiveRefCountsResult struct { } func RepairChunkLiveRefCountsResultWithDB(dbconn *sql.DB) (result RepairChunkLiveRefCountsResult, err error) { + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return RepairChunkLiveRefCountsResultWithDBContext(ctx, dbconn) +} + +func RepairChunkLiveRefCountsResultWithDBContext(ctx context.Context, dbconn *sql.DB) (result RepairChunkLiveRefCountsResult, err error) { if dbconn == nil { return RepairChunkLiveRefCountsResult{}, fmt.Errorf("db connection is nil") } - - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return RepairChunkLiveRefCountsResult{}, err + } tx, err := dbconn.BeginTx(ctx, nil) if err != nil { diff --git a/internal/maintenance/repair_refcounts.go b/internal/maintenance/repair_refcounts.go index af83d621..ebfc2df7 100644 --- a/internal/maintenance/repair_refcounts.go +++ b/internal/maintenance/repair_refcounts.go @@ -21,12 +21,21 @@ type RepairLogicalRefCountsResult struct { } func RepairLogicalRefCountsResultWithDB(dbconn *sql.DB) (result RepairLogicalRefCountsResult, err error) { + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return RepairLogicalRefCountsResultWithDBContext(ctx, dbconn) +} + +func RepairLogicalRefCountsResultWithDBContext(ctx context.Context, dbconn *sql.DB) (result RepairLogicalRefCountsResult, err error) { if dbconn == nil { return RepairLogicalRefCountsResult{}, fmt.Errorf("db connection is nil") } - - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return RepairLogicalRefCountsResult{}, err + } tx, err := dbconn.BeginTx(ctx, nil) if err != nil { From ccf9040b8f404556b4ff58365b9df70b11252027 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 15:20:09 +0200 Subject: [PATCH 17/27] Activate engine-owned recovery orchestration --- cmd/coldkeep/main.go | 11 +- cmd/coldkeep/recovery_engine_routing_test.go | 50 +++++++++ cmd/coldkeep/stored_path_engine_adapters.go | 33 ++++++ .../stored_path_engine_routing_test.go | 8 ++ docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- .../v1.13.12-phase16-recovery-activation.md | 64 +++++++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 15 ++- internal/engine/candidates.go | 46 +++----- internal/engine/candidates_test.go | 62 +---------- internal/engine/contracts_test.go | 31 ++---- internal/engine/engine.go | 3 + internal/engine/recover.go | 43 ++++++++ .../engine/recover_backend_contract_test.go | 101 ++++++++++++++++++ internal/recovery/system_recovery.go | 87 ++++++++++++--- 16 files changed, 424 insertions(+), 144 deletions(-) create mode 100644 cmd/coldkeep/recovery_engine_routing_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase16-recovery-activation.md create mode 100644 internal/engine/recover.go create mode 100644 internal/engine/recover_backend_contract_test.go diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index d810f080..09d1b588 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -186,12 +186,9 @@ const doctorDefaultVerifyLevel = verify.VerifyStandard const doctorOperationalHint = "After significant operations, run coldkeep doctor to validate system health." -// Current intentional CLI/domain ownership: doctor owns corrective recovery -// orchestration directly through lower-layer recovery and verify hooks. Phase -// 14 made this boundary decision, and the Phase 16 honesty proof confirmed the -// seam remains outside active engine ownership. Any future activation belongs -// to an explicit early-v2.0 design; these direct hooks are intentional. -var doctorRecoveryPhase = recovery.SystemRecoveryReportWithContainersDir +// Doctor's recovery stage uses the same active engine operation as startup. +// Phase 17 owns the remaining schema/verify/audit composition. +var doctorRecoveryPhase = runRecoveryThroughEngine var doctorSchemaVersionPhase = db.QueryCurrentSchemaVersion var doctorVerifyPhase = maintenance.VerifyCommandWithContainersDir var doctorSystemAuditPhase = maintenance.CollectSystemAuditSummary @@ -335,7 +332,7 @@ var runGCPhase = func(dryRun bool, containersDir string) (maintenance.GCResult, RetainedSharedLogical: result.SharedRetainedLogicalFiles, }, nil } -var startupRecoveryPhase = recovery.SystemRecoveryReportWithContainersDir +var startupRecoveryPhase = runRecoveryThroughEngine var loadDefaultStorageContextPhase = storage.LoadDefaultStorageContext // Compatibility-only direct snapshot-domain create seam retained for lower- diff --git a/cmd/coldkeep/recovery_engine_routing_test.go b/cmd/coldkeep/recovery_engine_routing_test.go new file mode 100644 index 00000000..8a0ec7b2 --- /dev/null +++ b/cmd/coldkeep/recovery_engine_routing_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "database/sql" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" +) + +func TestStartupAndDoctorRecoveryUseSameEngineOperation(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + calls := 0 + installRecoveryCommandStubs(t, dbconn, stubCommandEngine{recoverFunc: func(_ context.Context, req engine.RecoverRequest) (engine.RecoverResult, error) { + calls++ + if req != (engine.RecoverRequest{}) { + t.Fatalf("unexpected Recover request: %+v", req) + } + return engine.RecoverResult{AbortedLogicalFiles: int64(calls), SealingCompleted: 2}, nil + }}) + + startup, err := startupRecoveryPhase("/isolated/startup") + if err != nil || startup.AbortedLogicalFiles != 1 || startup.SealingCompleted != 2 { + t.Fatalf("startup recovery=(%+v,%v)", startup, err) + } + explicit, err := doctorRecoveryPhase("/isolated/doctor") + if err != nil || explicit.AbortedLogicalFiles != 2 || explicit.SealingCompleted != 2 { + t.Fatalf("doctor recovery=(%+v,%v)", explicit, err) + } + if calls != 2 { + t.Fatalf("Recover calls=%d want 2", calls) + } +} + +func installRecoveryCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { + t.Helper() + originalConnect := connectRecoveryDBPhase + originalNewEngine := newRecoveryCommandEngine + t.Cleanup(func() { + connectRecoveryDBPhase = originalConnect + newRecoveryCommandEngine = originalNewEngine + }) + connectRecoveryDBPhase = func() (*sql.DB, error) { return dbconn, nil } + newRecoveryCommandEngine = func(_ *sql.DB, containersDir string) (engine.Engine, error) { + if containersDir != "/isolated/startup" && containersDir != "/isolated/doctor" { + t.Fatalf("unexpected recovery container dir %q", containersDir) + } + return stub, nil + } +} diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index cb8c2358..dde98365 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -11,6 +11,7 @@ import ( "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/invariants" + "github.com/franchoy/coldkeep/internal/recovery" "github.com/franchoy/coldkeep/internal/storage" ) @@ -58,6 +59,38 @@ var newRepairCommandEngine = func(dbconn *sql.DB) (engine.Engine, error) { var connectRepairDBPhase = db.ConnectDB +var connectRecoveryDBPhase = db.ConnectDB + +var newRecoveryCommandEngine = func(dbconn *sql.DB, containersDir string) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn, ContainerDir: containersDir}) +} + +func runRecoveryThroughEngine(containersDir string) (recovery.Report, error) { + dbconn, err := connectRecoveryDBPhase() + if err != nil { + return recovery.Report{}, fmt.Errorf("failed to connect to DB: %w", err) + } + defer func() { _ = dbconn.Close() }() + eng, err := newRecoveryCommandEngine(dbconn, containersDir) + if err != nil { + return recovery.Report{}, err + } + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + result, err := eng.Recover(ctx, engine.RecoverRequest{}) + return recoveryReportFromEngine(result), err +} + +func recoveryReportFromEngine(result engine.RecoverResult) recovery.Report { + return recovery.Report{ + AbortedLogicalFiles: result.AbortedLogicalFiles, AbortedChunks: result.AbortedChunks, + QuarantinedMissing: result.QuarantinedMissing, QuarantinedCorruptTail: result.QuarantinedCorruptTail, + QuarantinedOrphan: result.QuarantinedOrphan, SkippedDirEntries: result.SkippedDirEntries, + CheckedContainerRecord: result.CheckedContainerRecord, CheckedDiskFiles: result.CheckedDiskFiles, + SealingCompleted: result.SealingCompleted, SealingQuarantined: result.SealingQuarantined, + } +} + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index 2b0d17df..a9133103 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -35,6 +35,7 @@ type stubCommandEngine struct { inspectFunc func(context.Context, engine.InspectRequest) (engine.InspectResult, error) verifyFunc func(context.Context, engine.VerifyRequest) (engine.VerifyResult, error) repairFunc func(context.Context, engine.RepairRequest) (engine.RepairResult, error) + recoverFunc func(context.Context, engine.RecoverRequest) (engine.RecoverResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -163,6 +164,13 @@ func (s stubCommandEngine) Repair(ctx context.Context, req engine.RepairRequest) return engine.RepairResult{}, errors.New("unexpected Repair call") } +func (s stubCommandEngine) Recover(ctx context.Context, req engine.RecoverRequest) (engine.RecoverResult, error) { + if s.recoverFunc != nil { + return s.recoverFunc(ctx, req) + } + return engine.RecoverResult{}, errors.New("unexpected Recover call") +} + func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 83397b59..41872ac3 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–15 are complete; Phase 16 is next. + 0–16 are complete; Phase 17 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index b980abba..50c77c23 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -105,11 +105,13 @@ ## Phase 16 — Recovery Activation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 16 recovery activation](v1.13.12-phase16-recovery-activation.md). ## Phase 17 — Doctor Activation -**Status:** Not started +**Status:** Next ## Phase 18 — Application Composition and Thin CLI Closure diff --git a/docs/release/v1.13/v1.13.12-phase16-recovery-activation.md b/docs/release/v1.13/v1.13.12-phase16-recovery-activation.md new file mode 100644 index 00000000..00c5ac1f --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase16-recovery-activation.md @@ -0,0 +1,64 @@ +# Coldkeep v1.13.12 Phase 16 — Recovery Activation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `16 — Recovery Activation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Production ownership + +`Engine.Recover` is now the active corrective recovery boundary. Its request +is intentionally empty: recovery is real mutation only, with no unsupported +dry-run, input, or limit facade. Its neutral result preserves all ten existing +recovery counters as exact 64-bit integers. + +The recovery package now exposes a context-aware, DB-injected entry point. The +legacy environment-opening entry point remains for non-engine compatibility +and delegates to the same implementation. One caller context now reaches every +database stage: processing logical-file abort, processing chunk abort, sealing +recovery, missing-container quarantine, corrupt-tail quarantine, and orphan +container reconciliation. + +Automatic startup recovery and Doctor's explicit recovery stage both use one +CLI adapter that opens the configured database, constructs an injected engine, +executes `Recover`, and projects the neutral result into the frozen recovery +report. Startup log buffering/event rendering remains outside the engine. +Doctor's schema, verification, audit, short-circuit, and rendering sequence is +unchanged and remains the explicit Phase 17 work. + +## Compatibility + +The outer coordination lease is still acquired before startup recovery. +Recovery stage order, SQL mutations, row counters, quarantine/sealing logic, +strict-recovery policy, filesystem checks, journal/restart behavior, +idempotence, healthy-log suppression, startup text/JSON events, Doctor report, +error text, and recovery exit classification remain unchanged. No schema, +repository, storage, backend-default, lock, or coordination change was made. + +## Validation + +- Direct engine corrective, idempotence, fault, and cancellation contracts: + pass on SQLite and PostgreSQL. +- Direct startup/Doctor shared-engine routing and result projection: pass. +- Existing recovery fault-filesystem, startup, Doctor, coordination, + integration, and adversarial suites: pass. +- Focused recovery, engine, and CLI race suites: pass. +- Full `go test -count=1 ./...`: pass. +- Release-state validator: development state, 1.13.12, zero violations after + tracker reconciliation. +- Validation root: `/tmp/coldkeep-v11312-phase16.k6jtmV/`. + +PostgreSQL validation created and dropped exactly: + +- `coldkeep_bt_testenginerecoveracrossbackendsandisidempo_32a3d1_1` +- `coldkeep_bt_testenginerecoverfaultandcancellationacros_32a3d1_2` + +A read-only maintenance query confirmed both exact names are absent. No +ordinary database was reset, migrated, truncated, or dropped. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 17 is authorized to activate `Engine.Doctor` as the ordered recovery → +schema → verify → audit composite and remove CLI-owned business sequencing +without changing report or first-failure behavior. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 1dedd7bd..f7d0049e 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–15 are Complete. Phase 16 is Next: activate one DB-injected recovery -engine operation and route explicit and startup recovery through it without -changing recovery ordering, counters, journal handling, or idempotence. +Phases 0–16 are Complete. Phase 17 is Next: activate `Engine.Doctor` as the +ordered recovery → schema → verify → audit composite while preserving its +first-failure short circuit and stable report projection. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index dd4d8762..51abe160 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -243,11 +243,22 @@ ## Phase 16 — Recovery Activation -**Phase status:** Next +**Phase status:** Complete + +- [x] `Engine.Recover` is an active, option-free corrective operation over the + injected database and configured container directory. +- [x] The recovery package exposes one context-aware DB-injected entry point; + its environment-opening compatibility entry delegates to it. +- [x] Automatic startup recovery and Doctor's explicit recovery stage use the + same engine operation and retain outer lease ordering and presentation. +- [x] Processing-row aborts, quarantine/sealing counters, fault ordering, + idempotence, cancellation, logging, and error/exit projection remain stable. +- [x] Direct engine, CLI routing, recovery, race, full regression, and shared + SQLite/PostgreSQL contract suites pass. ## Phase 17 — Doctor Activation -**Phase status:** Not started +**Phase status:** Next ## Phase 18 — Application Composition and Thin CLI Closure diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index f300349f..1be4cfc3 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -866,41 +866,23 @@ type RepairResult struct { // Recovery // --------------------------------------------------------------------------- -// RecoverRequest is a candidate-only request contract for a future corrective -// Recover operation. Recover is not a method on the current Engine interface. -// -// Request/result presence must not be mistaken for active engine ownership. -// Phase 14 and the Phase 16 honesty proof confirmed current CLI/domain -// ownership. Any early-v2.0 activation design must be explicit and -// behavior-preserving. -// // Safety invariant: Recovery must not legitimize corrupt mappings. Recovery is // a corrective integrity pass (abort dangling writes, clear stale sealing -// markers, quarantine corrupt/orphaned data), NOT a restore. The previous -// placeholder modeled it like a restore; that was incorrect. -type RecoverRequest struct { - // DryRun reports what recovery would do without mutating. - DryRun bool -} +// markers, quarantine corrupt/orphaned data), not a restore or simulation. +// RecoverRequest deliberately has no unsupported dry-run/input/limit surface. +type RecoverRequest struct{} -// RecoverResult is a candidate-only result contract for a future Recover -// operation. Recover is not a method on the current Engine interface. -// Phase 14 and the Phase 16 honesty proof confirmed current CLI/domain -// ownership; any early-v2.0 activation design must be explicit and -// behavior-preserving. -// -// Fields mirror the existing recovery report so the corrective outcome can be -// represented without CLI rendering. +// RecoverResult is the neutral corrective recovery report. type RecoverResult struct { - AbortedLogicalFiles int - AbortedChunks int - QuarantinedMissing int - QuarantinedCorruptTail int - QuarantinedOrphan int - SkippedDirEntries int - CheckedContainerRecord int - CheckedDiskFiles int - SealingCompleted int - SealingQuarantined int + AbortedLogicalFiles int64 + AbortedChunks int64 + QuarantinedMissing int64 + QuarantinedCorruptTail int64 + QuarantinedOrphan int64 + SkippedDirEntries int64 + CheckedContainerRecord int64 + CheckedDiskFiles int64 + SealingCompleted int64 + SealingQuarantined int64 Warnings []OperationWarning } diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index 1da83cb0..5b632d96 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Recover", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) @@ -41,8 +41,8 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { } // TestEngineActiveInterfaceExcludesStillInactiveOperations proves the active -// Engine interface still excludes corrective integrity operations that remain -// future-only. +// Engine interface does not acquire unapproved planning or snapshot-corrective +// aliases alongside the active Repair and Recover operations. func TestEngineActiveInterfaceExcludesStillInactiveOperations(t *testing.T) { typ := reflect.TypeOf((*engine.Engine)(nil)).Elem() @@ -54,7 +54,6 @@ func TestEngineActiveInterfaceExcludesStillInactiveOperations(t *testing.T) { for _, forbidden := range []string{ "SnapshotRepair", "RepairPlan", - "Recover", "SnapshotRecover", "RecoveryPlan", } { @@ -64,61 +63,6 @@ func TestEngineActiveInterfaceExcludesStillInactiveOperations(t *testing.T) { } } -// TestCandidateOnlyOperationContractsRemainOutsideActiveEngineOwnership -// documents which request/result pairs remain intentionally future-only and -// ties them to the approved active engine method set. -func TestCandidateOnlyOperationContractsRemainOutsideActiveEngineOwnership(t *testing.T) { - activeMethods := activeEngineMethodSet() - for _, tc := range candidateOnlyOperationCases() { - t.Run(tc.name, func(t *testing.T) { - if activeMethods[tc.forbiddenMethod] { - t.Fatalf("candidate-only contract %q unexpectedly has active engine method %q", tc.name, tc.forbiddenMethod) - } - - reqType := reflect.TypeOf(tc.requestType) - resType := reflect.TypeOf(tc.resultType) - if reqType.Name() == "" || resType.Name() == "" { - t.Fatalf("candidate-only contract %q must remain a named request/result type", tc.name) - } - if reqType.PkgPath() != "github.com/franchoy/coldkeep/internal/engine" || resType.PkgPath() != "github.com/franchoy/coldkeep/internal/engine" { - t.Fatalf("candidate-only contract %q moved outside engine package unexpectedly", tc.name) - } - if tc.futureDisposition == "" { - t.Fatalf("candidate-only contract %q must record a future disposition", tc.name) - } - }) - } -} - -type candidateOnlyOperationCase struct { - name string - requestType any - resultType any - forbiddenMethod string - futureDisposition string -} - -func candidateOnlyOperationCases() []candidateOnlyOperationCase { - return []candidateOnlyOperationCase{ - { - name: "recover", - requestType: engine.RecoverRequest{}, - resultType: engine.RecoverResult{}, - forbiddenMethod: "Recover", - futureDisposition: "early v2.0 activation-design decision", - }, - } -} - -func activeEngineMethodSet() map[string]bool { - typ := reflect.TypeOf((*engine.Engine)(nil)).Elem() - methods := make(map[string]bool, typ.NumMethod()) - for i := 0; i < typ.NumMethod(); i++ { - methods[typ.Method(i).Name] = true - } - return methods -} - // TestEngineContractTypesAreRendererNeutral verifies that engine contract // request/result types do not expose renderer-specific or // CLI-specific concepts (cobra, command, renderer, writer, stdout, stderr). diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index b0628c48..5e79cbab 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -21,7 +21,7 @@ func allEngineContractTypes() []struct { activeCoreContractTypes()..., ) types = append(types, activeSnapshotMutationContractTypes()...) - return append(types, candidateCorrectiveContractTypes()...) + return types } func activeCoreContractTypes() []struct { @@ -71,6 +71,8 @@ func activeCoreContractTypes() []struct { {"RepairRequest", engine.RepairRequest{}}, {"RepairTargetResult", engine.RepairTargetResult{}}, {"RepairResult", engine.RepairResult{}}, + {"RecoverRequest", engine.RecoverRequest{}}, + {"RecoverResult", engine.RecoverResult{}}, {"SnapshotMeta", engine.SnapshotMeta{}}, {"SnapshotListRequest", engine.SnapshotListRequest{}}, {"SnapshotListResult", engine.SnapshotListResult{}}, @@ -131,19 +133,6 @@ func activeSnapshotMutationContractTypes() []struct { } } -func candidateCorrectiveContractTypes() []struct { - name string - val any -} { - return []struct { - name string - val any - }{ - {"RecoverRequest", engine.RecoverRequest{}}, - {"RecoverResult", engine.RecoverResult{}}, - } -} - // allowedFieldPackages lists the package import paths a contract field type may // originate from. Contracts must stay backend- and renderer-neutral: only // engine-local types and the standard time package are permitted. Anything else @@ -626,10 +615,7 @@ func TestRepairContractRepresentsTargetsAndBatch(t *testing.T) { // TestRecoverContractRepresentsCorrectiveReport proves the recovery contract // models a corrective integrity report rather than a restore. func TestRecoverContractRepresentsCorrectiveReport(t *testing.T) { - req := engine.RecoverRequest{DryRun: true} - if !req.DryRun { - t.Error("recover dry-run not representable") - } + req := engine.RecoverRequest{} res := engine.RecoverResult{ AbortedLogicalFiles: 1, AbortedChunks: 2, @@ -645,13 +631,8 @@ func TestRecoverContractRepresentsCorrectiveReport(t *testing.T) { if res.AbortedLogicalFiles != 1 || res.SealingQuarantined != 10 { t.Fatalf("recovery report not representable: %+v", res) } - assertRecoverRequestIsCorrectiveOnly(t, req) -} - -func assertRecoverRequestIsCorrectiveOnly(t *testing.T, req engine.RecoverRequest) { - t.Helper() - if !req.DryRun { - t.Error("recover request should represent corrective dry-run mode") + if req != (engine.RecoverRequest{}) { + t.Fatalf("recover request unexpectedly carries options: %+v", req) } } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 96eb70dc..02976b8d 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -100,6 +100,9 @@ type Engine interface { // integrity recomputations. Each target remains transactionally independent; // FailFast stops after the first execution failure. Repair(ctx context.Context, req RepairRequest) (RepairResult, error) + // Recover executes the corrective repository recovery pass against the + // injected database and configured container directory. + Recover(ctx context.Context, req RecoverRequest) (RecoverResult, error) // Remove removes logical files from the repository by logical file ID. // Safety invariant: Remove must never make valid data unrecoverable. diff --git a/internal/engine/recover.go b/internal/engine/recover.go new file mode 100644 index 00000000..716e1cd5 --- /dev/null +++ b/internal/engine/recover.go @@ -0,0 +1,43 @@ +package engine + +import ( + "context" + "fmt" + + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/recovery" +) + +// Recover executes real corrective recovery. It intentionally has no dry-run +// mode: startup and explicit recovery share this exact operation. +func (e *DefaultEngine) Recover(ctx context.Context, _ RecoverRequest) (RecoverResult, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return RecoverResult{}, TranslateError("recover", err) + } + if e == nil || e.config.DB == nil { + return RecoverResult{}, TranslateErrorAs("recover", ErrorRecoveryFailed, fmt.Errorf("recover requires injected database")) + } + containersDir := e.config.ContainerDir + if containersDir == "" { + containersDir = container.ContainersDir + } + report, err := recovery.SystemRecoveryReportWithDBContext(ctx, e.config.DB, containersDir) + result := recoverResultFromReport(report) + if err != nil { + return result, TranslateErrorAs("recover", ErrorRecoveryFailed, err) + } + return result, nil +} + +func recoverResultFromReport(report recovery.Report) RecoverResult { + return RecoverResult{ + AbortedLogicalFiles: report.AbortedLogicalFiles, AbortedChunks: report.AbortedChunks, + QuarantinedMissing: report.QuarantinedMissing, QuarantinedCorruptTail: report.QuarantinedCorruptTail, + QuarantinedOrphan: report.QuarantinedOrphan, SkippedDirEntries: report.SkippedDirEntries, + CheckedContainerRecord: report.CheckedContainerRecord, CheckedDiskFiles: report.CheckedDiskFiles, + SealingCompleted: report.SealingCompleted, SealingQuarantined: report.SealingQuarantined, + } +} diff --git a/internal/engine/recover_backend_contract_test.go b/internal/engine/recover_backend_contract_test.go new file mode 100644 index 00000000..37db57b0 --- /dev/null +++ b/internal/engine/recover_backend_contract_test.go @@ -0,0 +1,101 @@ +package engine_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + filestate "github.com/franchoy/coldkeep/internal/status" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineRecoverAcrossBackendsAndIsIdempotent(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + logicalID, chunkID := seedProcessingRecoveryRows(t, backend) + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + + result, err := eng.Recover(context.Background(), engine.RecoverRequest{}) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if result.AbortedLogicalFiles != 1 || result.AbortedChunks != 1 || result.QuarantinedMissing != 0 || result.QuarantinedOrphan != 0 { + t.Fatalf("first recovery result=%+v", result) + } + assertRecoveryStatuses(t, backend, logicalID, chunkID) + + again, err := eng.Recover(context.Background(), engine.RecoverRequest{}) + if err != nil { + t.Fatalf("second Recover: %v", err) + } + if again.AbortedLogicalFiles != 0 || again.AbortedChunks != 0 || again.QuarantinedMissing != 0 || again.QuarantinedOrphan != 0 { + t.Fatalf("idempotent recovery result=%+v", again) + } + }) +} + +func TestEngineRecoverFaultAndCancellationAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + if _, err := eng.Recover(cancelled, engine.RecoverRequest{}); !errors.Is(err, context.Canceled) || !engine.IsCode(err, engine.ErrorCancelled) { + t.Fatalf("cancelled Recover error=%v code=%q", err, engine.CodeOf(err)) + } + + badRoot := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(badRoot, []byte("x"), 0o600); err != nil { + t.Fatalf("create recovery fault fixture: %v", err) + } + faulty, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: badRoot}) + if err != nil { + t.Fatalf("new faulty engine: %v", err) + } + if _, err := faulty.Recover(context.Background(), engine.RecoverRequest{}); err == nil || !engine.IsCode(err, engine.ErrorRecoveryFailed) { + t.Fatalf("faulty Recover error=%v code=%q", err, engine.CodeOf(err)) + } + }) +} + +func seedProcessingRecoveryRows(t *testing.T, backend backendtest.Backend) (int64, int64) { + t.Helper() + var logicalID int64 + if err := backend.DB.QueryRow( + `INSERT INTO logical_file (original_name, total_size, file_hash, status, ref_count, chunker_version) + VALUES ($1, 0, $2, $3, 0, 'v1-simple-rolling') RETURNING id`, + "recover-processing.bin", "recover-processing-logical-hash", filestate.LogicalFileProcessing, + ).Scan(&logicalID); err != nil { + t.Fatalf("insert processing logical file: %v", err) + } + var chunkID int64 + if err := backend.DB.QueryRow( + `INSERT INTO chunk (chunk_hash, size, status, live_ref_count, pin_count, retry_count, chunker_version) + VALUES ($1, 1, $2, 0, 0, 0, 'v1-simple-rolling') RETURNING id`, + "recover-processing-chunk-hash", filestate.ChunkProcessing, + ).Scan(&chunkID); err != nil { + t.Fatalf("insert processing chunk: %v", err) + } + return logicalID, chunkID +} + +func assertRecoveryStatuses(t *testing.T, backend backendtest.Backend, logicalID, chunkID int64) { + t.Helper() + var logicalStatus, chunkStatus string + if err := backend.DB.QueryRow(`SELECT status FROM logical_file WHERE id = $1`, logicalID).Scan(&logicalStatus); err != nil { + t.Fatalf("read logical status: %v", err) + } + if err := backend.DB.QueryRow(`SELECT status FROM chunk WHERE id = $1`, chunkID).Scan(&chunkStatus); err != nil { + t.Fatalf("read chunk status: %v", err) + } + if logicalStatus != filestate.LogicalFileAborted || chunkStatus != filestate.ChunkAborted { + t.Fatalf("recovered statuses=(%q,%q), want (%q,%q)", logicalStatus, chunkStatus, filestate.LogicalFileAborted, filestate.ChunkAborted) + } +} diff --git a/internal/recovery/system_recovery.go b/internal/recovery/system_recovery.go index 82e6f12a..e2f9ce61 100644 --- a/internal/recovery/system_recovery.go +++ b/internal/recovery/system_recovery.go @@ -71,35 +71,53 @@ func SystemRecoveryWithContainersDir(containersDir string) error { // changed row counts in the returned Report as expected, successful corrective // outcomes, not as errors. func SystemRecoveryReportWithContainersDir(containersDir string) (Report, error) { - stats := &recoveryStats{} - logRecoveryEvent("start", "containers_dir="+containersDir) dbconn, err := db.ConnectDB() if err != nil { - return buildReport(stats), fmt.Errorf("failed to connect to DB: %w", err) + return Report{}, fmt.Errorf("failed to connect to DB: %w", err) } defer func() { _ = dbconn.Close() }() + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return SystemRecoveryReportWithDBContext(ctx, dbconn, containersDir) +} + +// SystemRecoveryReportWithDBContext executes corrective recovery with a +// caller-owned database and operation context. The caller retains connection +// lifetime ownership. +func SystemRecoveryReportWithDBContext(ctx context.Context, dbconn *sql.DB, containersDir string) (Report, error) { + stats := &recoveryStats{} + if dbconn == nil { + return buildReport(stats), fmt.Errorf("db connection is nil") + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return buildReport(stats), err + } + logRecoveryEvent("start", "containers_dir="+containersDir) - err = abortProcessingLogicalFiles(dbconn, stats) + err := abortProcessingLogicalFilesWithContext(ctx, dbconn, stats) if err != nil { return buildReport(stats), err } - err = abortProcessingChunks(dbconn, stats) + err = abortProcessingChunksWithContext(ctx, dbconn, stats) if err != nil { return buildReport(stats), err } - err = recoverSealingContainers(dbconn, containersDir, stats) + err = recoverSealingContainersWithContext(ctx, dbconn, containersDir, stats) if err != nil { return buildReport(stats), err } - err = quarantineMissingContainers(dbconn, containersDir, stats) + err = quarantineMissingContainersWithContext(ctx, dbconn, containersDir, stats) if err != nil { return buildReport(stats), err } - err = quarantineCorruptActiveContainerTails(dbconn, containersDir, stats) + err = quarantineCorruptActiveContainerTailsWithContext(ctx, dbconn, containersDir, stats) if err != nil { return buildReport(stats), err } - err = quarantineOrphanContainers(dbconn, containersDir, stats) + err = quarantineOrphanContainersWithContext(ctx, dbconn, containersDir, stats) if err != nil { return buildReport(stats), err } @@ -139,7 +157,10 @@ func buildReport(stats *recoveryStats) Report { func abortProcessingLogicalFiles(dbconn *sql.DB, stats *recoveryStats) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() + return abortProcessingLogicalFilesWithContext(ctx, dbconn, stats) +} +func abortProcessingLogicalFilesWithContext(ctx context.Context, dbconn *sql.DB, stats *recoveryStats) error { logRecoveryEvent("abort_processing_logical_files_start") result, err := dbconn.ExecContext(ctx, `UPDATE logical_file SET status = $1 WHERE status = $2`, filestate.LogicalFileAborted, filestate.LogicalFileProcessing) if err != nil { @@ -157,7 +178,10 @@ func abortProcessingLogicalFiles(dbconn *sql.DB, stats *recoveryStats) error { func abortProcessingChunks(dbconn *sql.DB, stats *recoveryStats) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() + return abortProcessingChunksWithContext(ctx, dbconn, stats) +} +func abortProcessingChunksWithContext(ctx context.Context, dbconn *sql.DB, stats *recoveryStats) error { logRecoveryEvent("abort_processing_chunks_start") result, err := dbconn.ExecContext(ctx, `UPDATE chunk SET status = $1 WHERE status = $2`, filestate.ChunkAborted, filestate.ChunkProcessing) if err != nil { @@ -173,13 +197,22 @@ func abortProcessingChunks(dbconn *sql.DB, stats *recoveryStats) error { } func recoverSealingContainers(dbconn *sql.DB, containersDir string, stats *recoveryStats) error { - return recoverSealingContainersWithFS(dbconn, containersDir, stats, fsx.Default()) + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return recoverSealingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) +} + +func recoverSealingContainersWithContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats) error { + return recoverSealingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) } func recoverSealingContainersWithFS(dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() + return recoverSealingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsys) +} +func recoverSealingContainersWithFSContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { logRecoveryEvent("recover_sealing_containers_start") // Clean up stale markers where a previous run sealed successfully but did not @@ -308,13 +341,22 @@ func quarantineMissingContainerIfNeeded(ctx context.Context, dbconn *sql.DB, id } func quarantineMissingContainers(dbconn *sql.DB, containersDir string, stats *recoveryStats) error { - return quarantineMissingContainersWithFS(dbconn, containersDir, stats, fsx.Default()) + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return quarantineMissingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) +} + +func quarantineMissingContainersWithContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats) error { + return quarantineMissingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) } func quarantineMissingContainersWithFS(dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() + return quarantineMissingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsys) +} +func quarantineMissingContainersWithFSContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { logRecoveryEvent("quarantine_missing_containers_start") rows, err := dbconn.QueryContext(ctx, `SELECT id, filename FROM container WHERE quarantine = FALSE`) if err != nil { @@ -353,13 +395,22 @@ func quarantineMissingContainersWithFS(dbconn *sql.DB, containersDir string, sta } func quarantineCorruptActiveContainerTails(dbconn *sql.DB, containersDir string, stats *recoveryStats) error { - return quarantineCorruptActiveContainerTailsWithFS(dbconn, containersDir, stats, fsx.Default()) + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return quarantineCorruptActiveContainerTailsWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) +} + +func quarantineCorruptActiveContainerTailsWithContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats) error { + return quarantineCorruptActiveContainerTailsWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) } func quarantineCorruptActiveContainerTailsWithFS(dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() + return quarantineCorruptActiveContainerTailsWithFSContext(ctx, dbconn, containersDir, stats, fsys) +} +func quarantineCorruptActiveContainerTailsWithFSContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { logRecoveryEvent("quarantine_corrupt_active_container_tails_start") rows, err := dbconn.QueryContext(ctx, ` @@ -500,12 +551,22 @@ func quarantineOneActiveCorruptTail(ctx context.Context, dbconn *sql.DB, id int6 } func quarantineOrphanContainers(dbconn *sql.DB, containersDir string, stats *recoveryStats) error { - return quarantineOrphanContainersWithFS(dbconn, containersDir, stats, fsx.Default()) + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return quarantineOrphanContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) +} + +func quarantineOrphanContainersWithContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats) error { + return quarantineOrphanContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) } func quarantineOrphanContainersWithFS(dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { ctx, cancel := db.NewOperationContext(context.Background()) defer cancel() + return quarantineOrphanContainersWithFSContext(ctx, dbconn, containersDir, stats, fsys) +} + +func quarantineOrphanContainersWithFSContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { backend := db.BackendFromDB(dbconn) var reusedCount int64 From 27f1fec322a4ea7e2898e2e6302566bafee5d98f Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 15:30:20 +0200 Subject: [PATCH 18/27] Activate engine-owned doctor orchestration --- .../cli_boundary_preservation_test.go | 10 +- cmd/coldkeep/doctor_engine_routing_test.go | 83 +++++++++++ cmd/coldkeep/legacy_doctor_seams_test.go | 102 +++++++++++++ cmd/coldkeep/main.go | 74 +++++----- cmd/coldkeep/stored_path_engine_adapters.go | 21 +++ .../stored_path_engine_routing_test.go | 8 ++ docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- .../v1.13.12-phase17-doctor-activation.md | 65 +++++++++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 16 ++- internal/engine/candidates.go | 50 +++++++ internal/engine/candidates_test.go | 2 +- internal/engine/contracts_test.go | 4 + internal/engine/default_engine.go | 4 + internal/engine/doctor.go | 135 ++++++++++++++++++ .../engine/doctor_backend_contract_test.go | 28 ++++ internal/engine/doctor_core_test.go | 123 ++++++++++++++++ internal/engine/engine.go | 3 + internal/maintenance/verify_command.go | 6 +- 20 files changed, 693 insertions(+), 55 deletions(-) create mode 100644 cmd/coldkeep/doctor_engine_routing_test.go create mode 100644 cmd/coldkeep/legacy_doctor_seams_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase17-doctor-activation.md create mode 100644 internal/engine/doctor.go create mode 100644 internal/engine/doctor_backend_contract_test.go create mode 100644 internal/engine/doctor_core_test.go diff --git a/cmd/coldkeep/cli_boundary_preservation_test.go b/cmd/coldkeep/cli_boundary_preservation_test.go index 8301b2df..f789ec8d 100644 --- a/cmd/coldkeep/cli_boundary_preservation_test.go +++ b/cmd/coldkeep/cli_boundary_preservation_test.go @@ -327,7 +327,7 @@ func TestRunRepairCommandDoesNotConstructEngine(t *testing.T) { } } -func TestRunDoctorCommandPreservesDirectRecoveryOwnership(t *testing.T) { +func TestRunDoctorCommandPreservesRecoveryFailureShortCircuitCompatibility(t *testing.T) { originalRecovery := doctorRecoveryPhase originalSchema := doctorSchemaVersionPhase originalVerify := doctorVerifyPhase @@ -361,7 +361,7 @@ func TestRunDoctorCommandPreservesDirectRecoveryOwnership(t *testing.T) { t.Fatalf("expected recovery exit code %d, got %d", exitRecovery, got) } if !recoveryCalled { - t.Fatal("expected doctor to preserve direct recovery ownership") + t.Fatal("expected the legacy compatibility adapter to invoke recovery") } if schemaCalled { t.Fatal("schema phase should not run after recovery failure") @@ -371,7 +371,7 @@ func TestRunDoctorCommandPreservesDirectRecoveryOwnership(t *testing.T) { } } -func TestRunDoctorCommandDoesNotConstructEngine(t *testing.T) { +func TestRunDoctorLegacyCompatibilityDoesNotUseGenericCommandEngine(t *testing.T) { originalRecovery := doctorRecoveryPhase originalSchema := doctorSchemaVersionPhase originalVerify := doctorVerifyPhase @@ -388,7 +388,7 @@ func TestRunDoctorCommandDoesNotConstructEngine(t *testing.T) { engineConstructed := false newCommandEngine = func(_ *sql.DB, _ string) (engine.Engine, error) { engineConstructed = true - t.Fatal("doctor should not construct an engine repair/recovery API") + t.Fatal("doctor compatibility adapter should not use the generic command-engine seam") return nil, nil } doctorRecoveryPhase = func(string) (recovery.Report, error) { @@ -412,6 +412,6 @@ func TestRunDoctorCommandDoesNotConstructEngine(t *testing.T) { t.Fatalf("runDoctorCommand: %v", err) } if engineConstructed { - t.Fatal("doctor unexpectedly constructed an engine") + t.Fatal("doctor unexpectedly used the generic command-engine seam") } } diff --git a/cmd/coldkeep/doctor_engine_routing_test.go b/cmd/coldkeep/doctor_engine_routing_test.go new file mode 100644 index 00000000..12689106 --- /dev/null +++ b/cmd/coldkeep/doctor_engine_routing_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" +) + +func TestRunDoctorCommandUsesOneEngineOperationAndPreservesJSON(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + calls := 0 + installDoctorCommandStubs(t, dbconn, stubCommandEngine{doctorFunc: func(_ context.Context, req engine.DoctorRequest) (engine.DoctorResult, error) { + calls++ + if req.VerifyLevel != "full" { + t.Fatalf("Doctor request=%+v", req) + } + return engine.DoctorResult{ + Recovery: engine.RecoverResult{AbortedLogicalFiles: 1, SealingCompleted: 2}, + VerifyLevel: "full", SchemaVersion: 8, + RecoveryStatus: "ok", SchemaStatus: "ok", VerifyStatus: "ok", + PhysicalAudit: engine.DoctorPhysicalAudit{LogicalRefCountMismatches: 3}, + SnapshotAudit: engine.DoctorSnapshotAudit{SnapshotFileRows: 4}, + }, nil + }}) + + output := captureStdout(t, func() { + if err := runDoctorCommand(parsedCommandLine{method: "doctor", flags: map[string][]string{"full": {"true"}}}, outputModeJSON); err != nil { + t.Fatalf("runDoctorCommand: %v", err) + } + }) + if calls != 1 { + t.Fatalf("Doctor calls=%d want 1", calls) + } + for _, want := range []string{`"command":"doctor"`, `"verify_level":"full"`, `"schema_version":8`, `"aborted_logical_files":1`, `"sealing_completed":2`} { + if !strings.Contains(output, want) { + t.Fatalf("Doctor JSON missing %q: %s", want, output) + } + } + if strings.Contains(output, "physicalAudit") || strings.Contains(output, "snapshotAudit") { + t.Fatalf("private audit summaries leaked into stable JSON: %s", output) + } +} + +func TestRunDoctorCommandPreservesStageExitClasses(t *testing.T) { + tests := []struct { + name string + stage engine.DoctorStage + exit int + }{ + {"recovery", engine.DoctorStageRecovery, exitRecovery}, + {"schema", engine.DoctorStageSchema, exitGeneral}, + {"verify", engine.DoctorStageVerify, exitVerify}, + {"audit", engine.DoctorStageAudit, exitVerify}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dbconn := openSnapshotRoutingDB(t) + installDoctorCommandStubs(t, dbconn, stubCommandEngine{doctorFunc: func(context.Context, engine.DoctorRequest) (engine.DoctorResult, error) { + return engine.DoctorResult{FailedStage: tc.stage}, errors.New("doctor " + tc.name + " phase failed") + }}) + err := runDoctorCommand(parsedCommandLine{method: "doctor", flags: map[string][]string{}}, outputModeText) + if err == nil || classifyExitCode(err) != tc.exit { + t.Fatalf("stage %s error=%v exit=%d want %d", tc.stage, err, classifyExitCode(err), tc.exit) + } + }) + } +} + +func installDoctorCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { + t.Helper() + originalConnect := connectDoctorDBPhase + originalNewEngine := newDoctorCommandEngine + t.Cleanup(func() { + connectDoctorDBPhase = originalConnect + newDoctorCommandEngine = originalNewEngine + }) + connectDoctorDBPhase = func() (*sql.DB, error) { return dbconn, nil } + newDoctorCommandEngine = func(*sql.DB, string) (engine.Engine, error) { return stub, nil } +} diff --git a/cmd/coldkeep/legacy_doctor_seams_test.go b/cmd/coldkeep/legacy_doctor_seams_test.go new file mode 100644 index 00000000..abb58f6b --- /dev/null +++ b/cmd/coldkeep/legacy_doctor_seams_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + + "github.com/franchoy/coldkeep/internal/container" + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/maintenance" + "github.com/franchoy/coldkeep/internal/recovery" + "github.com/franchoy/coldkeep/internal/verify" +) + +var doctorRecoveryPhase = runRecoveryThroughEngine +var doctorSchemaVersionPhase = db.QueryCurrentSchemaVersion +var doctorVerifyPhase = maintenance.VerifyCommandWithContainersDir +var doctorSystemAuditPhase = maintenance.CollectSystemAuditSummary + +var productionDoctorDBConnector = connectDoctorDBPhase +var productionDoctorCommandEngine = newDoctorCommandEngine + +func init() { + connectDoctorDBPhase = func() (*sql.DB, error) { return sql.Open("sqlite3", ":memory:") } + newDoctorCommandEngine = func(*sql.DB, string) (engine.Engine, error) { + return legacyDoctorTestEngine{}, nil + } +} + +type legacyDoctorTestEngine struct{ engine.Engine } + +func (legacyDoctorTestEngine) Doctor(_ context.Context, req engine.DoctorRequest) (engine.DoctorResult, error) { + level, err := legacyDoctorVerifyLevel(req.VerifyLevel) + if err != nil { + return engine.DoctorResult{}, err + } + result := engine.DoctorResult{VerifyLevel: req.VerifyLevel} + recoveryReport, err := doctorRecoveryPhase(container.ContainersDir) + result.Recovery = recoverResultForDoctorTest(recoveryReport) + if err != nil { + result.RecoveryStatus, result.FailedStage = "error", engine.DoctorStageRecovery + return result, fmt.Errorf("doctor recovery phase failed: %w", err) + } + result.RecoveryStatus = "ok" + result.SchemaVersion, err = doctorSchemaVersionPhase() + if err != nil { + result.SchemaStatus, result.FailedStage = "error", engine.DoctorStageSchema + return result, fmt.Errorf("doctor schema/version check failed: %w", err) + } + result.SchemaStatus = "ok" + if err := doctorVerifyPhase(container.ContainersDir, "system", 0, level); err != nil { + result.VerifyStatus, result.FailedStage = "error", engine.DoctorStageVerify + return result, fmt.Errorf("doctor verify phase failed: %w", err) + } + result.VerifyStatus = "ok" + audit, err := doctorSystemAuditPhase() + if err != nil { + result.FailedStage = engine.DoctorStageAudit + return result, fmt.Errorf("doctor audit summary phase failed: %w", err) + } + result.PhysicalAudit = engine.DoctorPhysicalAudit{ + OrphanPhysicalFileRows: audit.Physical.OrphanPhysicalFileRows, + LogicalRefCountMismatches: audit.Physical.LogicalRefCountMismatches, + NegativeLogicalRefCounts: audit.Physical.NegativeLogicalRefCounts, + } + result.SnapshotAudit = engine.DoctorSnapshotAudit{ + SnapshotFileRows: audit.Snapshot.SnapshotFileRows, + OrphanSnapshotPathRefs: audit.Snapshot.OrphanSnapshotPathRefs, + DuplicateSnapshotPathPairs: audit.Snapshot.DuplicateSnapshotPathPairs, + SnapshotReferencedLogicalFiles: audit.Snapshot.SnapshotReferencedLogicalFiles, + SnapshotOnlyLogicalFiles: audit.Snapshot.SnapshotOnlyLogicalFiles, + SharedLogicalFiles: audit.Snapshot.SharedLogicalFiles, + OrphanSnapshotLogicalRefs: audit.Snapshot.OrphanSnapshotLogicalRefs, + InvalidLifecycleStates: audit.Snapshot.InvalidSnapshotLifecycleStates, + RetainedMissingChunkGraph: audit.Snapshot.RetainedMissingChunkGraph, + } + return result, nil +} + +func legacyDoctorVerifyLevel(level string) (verify.VerifyLevel, error) { + switch level { + case "standard": + return verify.VerifyStandard, nil + case "full": + return verify.VerifyFull, nil + case "deep": + return verify.VerifyDeep, nil + default: + return 0, fmt.Errorf("invalid doctor verify level %q", level) + } +} + +func recoverResultForDoctorTest(report recovery.Report) engine.RecoverResult { + return engine.RecoverResult{ + AbortedLogicalFiles: report.AbortedLogicalFiles, AbortedChunks: report.AbortedChunks, + QuarantinedMissing: report.QuarantinedMissing, QuarantinedCorruptTail: report.QuarantinedCorruptTail, + QuarantinedOrphan: report.QuarantinedOrphan, SkippedDirEntries: report.SkippedDirEntries, + CheckedContainerRecord: report.CheckedContainerRecord, CheckedDiskFiles: report.CheckedDiskFiles, + SealingCompleted: report.SealingCompleted, SealingQuarantined: report.SealingQuarantined, + } +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index 09d1b588..adcde1e2 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -186,13 +186,6 @@ const doctorDefaultVerifyLevel = verify.VerifyStandard const doctorOperationalHint = "After significant operations, run coldkeep doctor to validate system health." -// Doctor's recovery stage uses the same active engine operation as startup. -// Phase 17 owns the remaining schema/verify/audit composition. -var doctorRecoveryPhase = runRecoveryThroughEngine -var doctorSchemaVersionPhase = db.QueryCurrentSchemaVersion -var doctorVerifyPhase = maintenance.VerifyCommandWithContainersDir -var doctorSystemAuditPhase = maintenance.CollectSystemAuditSummary - var storeByFilePhase = func(sgctx *storage.StorageContext, path, codecName string) (storage.StoreFileResult, error) { if sgctx == nil || sgctx.DB == nil { return storage.StoreFileResult{}, fmt.Errorf("store: storage context DB is required") @@ -2663,39 +2656,18 @@ func runDoctorCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return err } - report := doctorReport{ - VerifyLevel: verifyLevelToString(verifyLevel), - } - - recoveryReport, recoveryErr := doctorRecoveryPhase(container.ContainersDir) - report.Recovery = recoveryReport - if recoveryErr != nil { - report.RecoveryStatus = "error" - return recoveryError(fmt.Errorf("doctor recovery phase failed: %w", recoveryErr)) - } - report.RecoveryStatus = "ok" - - schemaVersion, schemaErr := doctorSchemaVersionPhase() - if schemaErr != nil { - report.SchemaStatus = "error" - return fmt.Errorf("doctor schema/version check failed: %w", schemaErr) - } - report.SchemaVersion = schemaVersion - report.SchemaStatus = "ok" - - verifyErr := doctorVerifyPhase(container.ContainersDir, "system", 0, verifyLevel) - if verifyErr != nil { - report.VerifyStatus = "error" - return verifyError(fmt.Errorf("doctor verify phase failed: %w", verifyErr)) - } - report.VerifyStatus = "ok" - - auditSummary, auditErr := doctorSystemAuditPhase() - if auditErr != nil { - return verifyError(fmt.Errorf("doctor audit summary phase failed: %w", auditErr)) + engineResult, doctorErr := executeDoctorEngine(container.ContainersDir, verifyLevelToString(verifyLevel)) + if doctorErr != nil { + switch engineResult.FailedStage { + case engine.DoctorStageRecovery: + return recoveryError(doctorErr) + case engine.DoctorStageVerify, engine.DoctorStageAudit: + return verifyError(doctorErr) + default: + return doctorErr + } } - report.physicalAudit = auditSummary.Physical - report.snapshotAudit = auditSummary.Snapshot + report := doctorReportFromEngine(engineResult) // Intentional JSON contract (frozen v1.0): // - Startup/preflight recovery diagnostics are emitted as stderr events @@ -2726,6 +2698,30 @@ func runDoctorCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return nil } +func doctorReportFromEngine(result engine.DoctorResult) doctorReport { + return doctorReport{ + Recovery: recoveryReportFromEngine(result.Recovery), VerifyLevel: result.VerifyLevel, + SchemaVersion: result.SchemaVersion, RecoveryStatus: result.RecoveryStatus, + VerifyStatus: result.VerifyStatus, SchemaStatus: result.SchemaStatus, + physicalAudit: verify.PhysicalFileIntegritySummary{ + OrphanPhysicalFileRows: result.PhysicalAudit.OrphanPhysicalFileRows, + LogicalRefCountMismatches: result.PhysicalAudit.LogicalRefCountMismatches, + NegativeLogicalRefCounts: result.PhysicalAudit.NegativeLogicalRefCounts, + }, + snapshotAudit: verify.SnapshotReachabilityIntegritySummary{ + SnapshotFileRows: result.SnapshotAudit.SnapshotFileRows, + OrphanSnapshotPathRefs: result.SnapshotAudit.OrphanSnapshotPathRefs, + DuplicateSnapshotPathPairs: result.SnapshotAudit.DuplicateSnapshotPathPairs, + SnapshotReferencedLogicalFiles: result.SnapshotAudit.SnapshotReferencedLogicalFiles, + SnapshotOnlyLogicalFiles: result.SnapshotAudit.SnapshotOnlyLogicalFiles, + SharedLogicalFiles: result.SnapshotAudit.SharedLogicalFiles, + OrphanSnapshotLogicalRefs: result.SnapshotAudit.OrphanSnapshotLogicalRefs, + InvalidSnapshotLifecycleStates: result.SnapshotAudit.InvalidLifecycleStates, + RetainedMissingChunkGraph: result.SnapshotAudit.RetainedMissingChunkGraph, + }, + } +} + func formatDoctorTextReport(report doctorReport) string { overallStatus := "ok" if report.RecoveryStatus != "ok" || report.VerifyStatus != "ok" || report.SchemaStatus != "ok" { diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index dde98365..3718d2c0 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -91,6 +91,27 @@ func recoveryReportFromEngine(result engine.RecoverResult) recovery.Report { } } +var connectDoctorDBPhase = db.ConnectDB + +var newDoctorCommandEngine = func(dbconn *sql.DB, containersDir string) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn, ContainerDir: containersDir}) +} + +func executeDoctorEngine(containersDir, verifyLevel string) (engine.DoctorResult, error) { + dbconn, err := connectDoctorDBPhase() + if err != nil { + return engine.DoctorResult{}, fmt.Errorf("failed to connect to DB: %w", err) + } + defer func() { _ = dbconn.Close() }() + eng, err := newDoctorCommandEngine(dbconn, containersDir) + if err != nil { + return engine.DoctorResult{}, err + } + ctx, cancel := db.NewOperationContext(context.Background()) + defer cancel() + return eng.Doctor(ctx, engine.DoctorRequest{VerifyLevel: verifyLevel}) +} + func restoreStoredPathWithEngine( ctx context.Context, eng engine.Engine, diff --git a/cmd/coldkeep/stored_path_engine_routing_test.go b/cmd/coldkeep/stored_path_engine_routing_test.go index a9133103..852fe788 100644 --- a/cmd/coldkeep/stored_path_engine_routing_test.go +++ b/cmd/coldkeep/stored_path_engine_routing_test.go @@ -36,6 +36,7 @@ type stubCommandEngine struct { verifyFunc func(context.Context, engine.VerifyRequest) (engine.VerifyResult, error) repairFunc func(context.Context, engine.RepairRequest) (engine.RepairResult, error) recoverFunc func(context.Context, engine.RecoverRequest) (engine.RecoverResult, error) + doctorFunc func(context.Context, engine.DoctorRequest) (engine.DoctorResult, error) } func (s stubCommandEngine) SnapshotCreate(ctx context.Context, req engine.SnapshotCreateRequest) (engine.SnapshotCreateResult, error) { @@ -171,6 +172,13 @@ func (s stubCommandEngine) Recover(ctx context.Context, req engine.RecoverReques return engine.RecoverResult{}, errors.New("unexpected Recover call") } +func (s stubCommandEngine) Doctor(ctx context.Context, req engine.DoctorRequest) (engine.DoctorResult, error) { + if s.doctorFunc != nil { + return s.doctorFunc(ctx, req) + } + return engine.DoctorResult{}, errors.New("unexpected Doctor call") +} + func TestRunStoreFolderCommandUsesEngineJSONParity(t *testing.T) { dbconn := openSnapshotRoutingDB(t) installStoreFolderCommandStubs(t, dbconn, stubCommandEngine{ diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 41872ac3..6986549c 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–16 are complete; Phase 17 is next. + 0–17 are complete; Phase 18 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 50c77c23..117b63f2 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -111,11 +111,13 @@ ## Phase 17 — Doctor Activation -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 17 Doctor activation](v1.13.12-phase17-doctor-activation.md). ## Phase 18 — Application Composition and Thin CLI Closure -**Status:** Not started +**Status:** Next ## Phase 19 — Dependency and Contract Hardening diff --git a/docs/release/v1.13/v1.13.12-phase17-doctor-activation.md b/docs/release/v1.13/v1.13.12-phase17-doctor-activation.md new file mode 100644 index 00000000..2996b26f --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase17-doctor-activation.md @@ -0,0 +1,65 @@ +# Coldkeep v1.13.12 Phase 17 — Doctor Activation + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `17 — Doctor Activation` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Production ownership + +`Engine.Doctor` now owns the ordered corrective health gate: recovery, schema, +system verification, then integrity audit. It reuses active `Engine.Recover` +and `Engine.Verify`, reads schema and audit truth through the same injected +database, and stops at the first failed stage. A neutral failed-stage value +lets compatibility adapters retain the established exit class without moving +business sequencing back into the CLI. + +The active request accepts only the supported standard, full, and deep verify +levels; invalid levels fail before corrective recovery. The result carries the +complete neutral recovery report, schema version, stable stage statuses, and +physical/snapshot audit summaries. The maintenance audit now has a public +DB-injected entry point for this engine-owned composite. + +Production CLI Doctor now parses arguments, opens the configured database, +issues one engine request, maps the failed stage to the frozen exit class, and +projects success into the existing private report. Audit summaries remain +excluded from the stable Doctor JSON while remaining available to the text +renderer exactly as before. Legacy lower-layer seams exist only in tests. + +## Compatibility + +Corrective ordering, first-failure short circuit, verify-level defaults, +recovery counters, schema version, audit counters, human report, JSON payload, +private audit-field behavior, hints, error text, and recovery/general/verify +exit codes remain unchanged. Doctor still emits no partial command payload on +failure. No schema, repository, storage, backend-default, lock, coordination, +or startup-recovery change was made. + +## Validation + +- Direct engine stage-order, all-stage short-circuit, invalid-before-mutation, + neutral-result, and typed-error suites: pass. +- Direct CLI one-operation routing, private JSON projection, and stage exit + classification suites: pass. +- Shared SQLite/PostgreSQL end-to-end Doctor contract: pass. +- Existing Doctor golden text/JSON, recovery, verification, audit, + coordination, integration, and adversarial suites: pass. +- Focused engine, maintenance, recovery, and CLI race suites: pass. +- Full `go test -count=1 ./...`: pass. +- Release-state validator: development state, 1.13.12, zero violations after + tracker reconciliation. +- Validation root: `/tmp/coldkeep-v11312-phase17.jdkQbh/`. + +PostgreSQL validation created and dropped exactly: + +- `coldkeep_bt_testenginedoctoracrossbackends_postgres_33a897_1` + +A read-only maintenance query confirmed the exact name is absent. No ordinary +database was reset, migrated, truncated, or dropped. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 18 is authorized to introduce the application composition/session +boundary, preserve outer coordination ordering, remove remaining production +CLI lower-layer calls, and enforce the final thin-CLI boundary statically. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index f7d0049e..c1247683 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–16 are Complete. Phase 17 is Next: activate `Engine.Doctor` as the -ordered recovery → schema → verify → audit composite while preserving its -first-failure short circuit and stable report projection. +Phases 0–17 are Complete. Phase 18 is Next: introduce the application +composition/session boundary and close the production CLI to parsing, +projection, rendering, coordination, and application opening only. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 51abe160..3384e98b 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -258,11 +258,23 @@ ## Phase 17 — Doctor Activation -**Phase status:** Next +**Phase status:** Complete + +- [x] `Engine.Doctor` owns the ordered recovery → schema → verify → audit + composite and stops at the same first failing stage. +- [x] Doctor reuses active `Engine.Recover` and `Engine.Verify`; schema and + audit read from the same injected database without duplicated correctness + logic. +- [x] Neutral recovery and audit DTOs preserve exact counters while private + CLI projection retains the frozen JSON and text report shapes. +- [x] Recovery, schema, verify, and audit failures preserve their established + recovery/general/verify exit classifications and emit no partial payload. +- [x] Direct stage-order, short-circuit, CLI routing, race, full regression, + and SQLite/PostgreSQL Doctor contract suites pass. ## Phase 18 — Application Composition and Thin CLI Closure -**Phase status:** Not started +**Phase status:** Next ## Phase 19 — Dependency and Contract Hardening diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index 1be4cfc3..db8b5820 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -886,3 +886,53 @@ type RecoverResult struct { SealingQuarantined int64 Warnings []OperationWarning } + +// DoctorRequest selects the verification strength for the corrective health +// gate. Empty VerifyLevel means standard. +type DoctorRequest struct { + VerifyLevel string +} + +// DoctorPhysicalAudit is the neutral current-path integrity summary. +type DoctorPhysicalAudit struct { + OrphanPhysicalFileRows int64 + LogicalRefCountMismatches int64 + NegativeLogicalRefCounts int64 +} + +// DoctorSnapshotAudit is the neutral snapshot-retention integrity summary. +type DoctorSnapshotAudit struct { + SnapshotFileRows int64 + OrphanSnapshotPathRefs int64 + DuplicateSnapshotPathPairs int64 + SnapshotReferencedLogicalFiles int64 + SnapshotOnlyLogicalFiles int64 + SharedLogicalFiles int64 + OrphanSnapshotLogicalRefs int64 + InvalidLifecycleStates int64 + RetainedMissingChunkGraph int64 +} + +// DoctorStage identifies one ordered Doctor stage. +type DoctorStage string + +const ( + DoctorStageRecovery DoctorStage = "recovery" + DoctorStageSchema DoctorStage = "schema" + DoctorStageVerify DoctorStage = "verify" + DoctorStageAudit DoctorStage = "audit" +) + +// DoctorResult is the presentation-neutral ordered recovery, schema, +// verification, and audit report. Status strings retain the stable CLI values. +type DoctorResult struct { + Recovery RecoverResult + VerifyLevel string + SchemaVersion int64 + RecoveryStatus string + VerifyStatus string + SchemaStatus string + PhysicalAudit DoctorPhysicalAudit + SnapshotAudit DoctorSnapshotAudit + FailedStage DoctorStage +} diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index 5b632d96..53bcc2b6 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Recover", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Recover", "Doctor", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 5e79cbab..2b57aac6 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -73,6 +73,10 @@ func activeCoreContractTypes() []struct { {"RepairResult", engine.RepairResult{}}, {"RecoverRequest", engine.RecoverRequest{}}, {"RecoverResult", engine.RecoverResult{}}, + {"DoctorRequest", engine.DoctorRequest{}}, + {"DoctorPhysicalAudit", engine.DoctorPhysicalAudit{}}, + {"DoctorSnapshotAudit", engine.DoctorSnapshotAudit{}}, + {"DoctorResult", engine.DoctorResult{}}, {"SnapshotMeta", engine.SnapshotMeta{}}, {"SnapshotListRequest", engine.SnapshotListRequest{}}, {"SnapshotListResult", engine.SnapshotListResult{}}, diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index 12861a2d..bf7334a2 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -52,6 +52,10 @@ type DefaultEngine struct { config Config obs *observability.Service snapshotIDGenerator snapshotIDGenerator + doctorRecover func(context.Context) (RecoverResult, error) + doctorSchema func(*sql.DB) (int64, error) + doctorVerify func(context.Context, string) error + doctorAudit func(*sql.DB) (DoctorPhysicalAudit, DoctorSnapshotAudit, error) } // New returns a new DefaultEngine with the given configuration. diff --git a/internal/engine/doctor.go b/internal/engine/doctor.go new file mode 100644 index 00000000..326cc97e --- /dev/null +++ b/internal/engine/doctor.go @@ -0,0 +1,135 @@ +package engine + +import ( + "context" + "fmt" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/invariants" + "github.com/franchoy/coldkeep/internal/maintenance" +) + +func (e *DefaultEngine) Doctor(ctx context.Context, req DoctorRequest) (DoctorResult, error) { + if ctx == nil { + ctx = context.Background() + } + level, err := normalizeDoctorVerifyLevel(req.VerifyLevel) + if err != nil { + return DoctorResult{}, TranslateErrorAs("doctor", ErrorInvalidArgument, err) + } + result := DoctorResult{VerifyLevel: level} + if err := ctx.Err(); err != nil { + return result, TranslateError("doctor", err) + } + if e == nil || e.config.DB == nil { + return result, doctorStageError(ErrorOperationFailed, "doctor requires injected database", fmt.Errorf("doctor requires injected database")) + } + + recoveryResult, err := e.runDoctorRecovery(ctx) + result.Recovery = recoveryResult + if err != nil { + result.RecoveryStatus = "error" + result.FailedStage = DoctorStageRecovery + code := ErrorRecoveryFailed + if CodeOf(err) == ErrorCancelled { + code = ErrorCancelled + } + return result, doctorStageError(code, fmt.Sprintf("doctor recovery phase failed: %v", err), err) + } + result.RecoveryStatus = "ok" + + schemaVersion, err := e.runDoctorSchema() + if err != nil { + result.SchemaStatus = "error" + result.FailedStage = DoctorStageSchema + return result, doctorStageError(ErrorOperationFailed, fmt.Sprintf("doctor schema/version check failed: %v", err), err) + } + result.SchemaVersion = schemaVersion + result.SchemaStatus = "ok" + + if err := e.runDoctorVerification(ctx, level); err != nil { + result.VerifyStatus = "error" + result.FailedStage = DoctorStageVerify + code := ErrorVerificationFailed + if CodeOf(err) == ErrorCancelled { + code = ErrorCancelled + } + return result, doctorStageError(code, fmt.Sprintf("doctor verify phase failed: %v", err), err) + } + result.VerifyStatus = "ok" + + physicalAudit, snapshotAudit, err := e.runDoctorAudit() + if err != nil { + result.FailedStage = DoctorStageAudit + return result, doctorStageError(ErrorVerificationFailed, fmt.Sprintf("doctor audit summary phase failed: %v", err), err) + } + result.PhysicalAudit = physicalAudit + result.SnapshotAudit = snapshotAudit + return result, nil +} + +func (e *DefaultEngine) runDoctorRecovery(ctx context.Context) (RecoverResult, error) { + if e.doctorRecover != nil { + return e.doctorRecover(ctx) + } + return e.Recover(ctx, RecoverRequest{}) +} + +func (e *DefaultEngine) runDoctorSchema() (int64, error) { + if e.doctorSchema != nil { + return e.doctorSchema(e.config.DB) + } + return db.CurrentSchemaVersion(e.config.DB) +} + +func (e *DefaultEngine) runDoctorVerification(ctx context.Context, level string) error { + if e.doctorVerify != nil { + return e.doctorVerify(ctx, level) + } + _, err := e.Verify(ctx, VerifyRequest{Target: "system", Level: level}) + return err +} + +func (e *DefaultEngine) runDoctorAudit() (DoctorPhysicalAudit, DoctorSnapshotAudit, error) { + if e.doctorAudit != nil { + return e.doctorAudit(e.config.DB) + } + audit, err := maintenance.CollectSystemAuditSummaryWithDB(e.config.DB) + if err != nil { + return DoctorPhysicalAudit{}, DoctorSnapshotAudit{}, err + } + return DoctorPhysicalAudit{ + OrphanPhysicalFileRows: audit.Physical.OrphanPhysicalFileRows, + LogicalRefCountMismatches: audit.Physical.LogicalRefCountMismatches, + NegativeLogicalRefCounts: audit.Physical.NegativeLogicalRefCounts, + }, DoctorSnapshotAudit{ + SnapshotFileRows: audit.Snapshot.SnapshotFileRows, + OrphanSnapshotPathRefs: audit.Snapshot.OrphanSnapshotPathRefs, + DuplicateSnapshotPathPairs: audit.Snapshot.DuplicateSnapshotPathPairs, + SnapshotReferencedLogicalFiles: audit.Snapshot.SnapshotReferencedLogicalFiles, + SnapshotOnlyLogicalFiles: audit.Snapshot.SnapshotOnlyLogicalFiles, + SharedLogicalFiles: audit.Snapshot.SharedLogicalFiles, + OrphanSnapshotLogicalRefs: audit.Snapshot.OrphanSnapshotLogicalRefs, + InvalidLifecycleStates: audit.Snapshot.InvalidSnapshotLifecycleStates, + RetainedMissingChunkGraph: audit.Snapshot.RetainedMissingChunkGraph, + }, nil +} + +func normalizeDoctorVerifyLevel(level string) (string, error) { + switch level { + case "": + return "standard", nil + case "standard", "full", "deep": + return level, nil + default: + return "", fmt.Errorf("invalid doctor verify level %q: must be standard, full, or deep", level) + } +} + +func doctorStageError(code ErrorCode, message string, cause error) error { + invariantCode := "" + if value, ok := invariants.Code(cause); ok { + invariantCode = value + } + return NewError(code, "doctor", message, invariantCode, cause) +} diff --git a/internal/engine/doctor_backend_contract_test.go b/internal/engine/doctor_backend_contract_test.go new file mode 100644 index 00000000..4d46bd01 --- /dev/null +++ b/internal/engine/doctor_backend_contract_test.go @@ -0,0 +1,28 @@ +package engine_test + +import ( + "context" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineDoctorAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + result, err := eng.Doctor(context.Background(), engine.DoctorRequest{}) + if err != nil { + t.Fatalf("Doctor: %v", err) + } + if result.VerifyLevel != "standard" || result.RecoveryStatus != "ok" || result.SchemaStatus != "ok" || result.VerifyStatus != "ok" || result.SchemaVersion <= 0 || result.FailedStage != "" { + t.Fatalf("Doctor result=%+v", result) + } + if result.PhysicalAudit != (engine.DoctorPhysicalAudit{}) || result.SnapshotAudit != (engine.DoctorSnapshotAudit{}) { + t.Fatalf("empty repository audits=%+v %+v", result.PhysicalAudit, result.SnapshotAudit) + } + }) +} diff --git a/internal/engine/doctor_core_test.go b/internal/engine/doctor_core_test.go new file mode 100644 index 00000000..fcf70628 --- /dev/null +++ b/internal/engine/doctor_core_test.go @@ -0,0 +1,123 @@ +package engine + +import ( + "context" + "database/sql" + "errors" + "reflect" + "strings" + "testing" +) + +func TestDoctorOwnsOrderedCompositeAndNeutralResult(t *testing.T) { + eng := newDoctorHookEngine(t) + var calls []string + eng.doctorRecover = func(context.Context) (RecoverResult, error) { + calls = append(calls, "recovery") + return RecoverResult{AbortedLogicalFiles: 2}, nil + } + eng.doctorSchema = func(*sql.DB) (int64, error) { + calls = append(calls, "schema") + return 8, nil + } + eng.doctorVerify = func(_ context.Context, level string) error { + calls = append(calls, "verify:"+level) + return nil + } + eng.doctorAudit = func(*sql.DB) (DoctorPhysicalAudit, DoctorSnapshotAudit, error) { + calls = append(calls, "audit") + return DoctorPhysicalAudit{LogicalRefCountMismatches: 3}, DoctorSnapshotAudit{SnapshotFileRows: 4}, nil + } + + result, err := eng.Doctor(context.Background(), DoctorRequest{VerifyLevel: "full"}) + if err != nil { + t.Fatalf("Doctor: %v", err) + } + if want := []string{"recovery", "schema", "verify:full", "audit"}; !reflect.DeepEqual(calls, want) { + t.Fatalf("stage order=%v want %v", calls, want) + } + if result.RecoveryStatus != "ok" || result.SchemaStatus != "ok" || result.VerifyStatus != "ok" || result.FailedStage != "" { + t.Fatalf("statuses=%+v", result) + } + if result.Recovery.AbortedLogicalFiles != 2 || result.SchemaVersion != 8 || result.PhysicalAudit.LogicalRefCountMismatches != 3 || result.SnapshotAudit.SnapshotFileRows != 4 { + t.Fatalf("neutral report=%+v", result) + } +} + +func TestDoctorShortCircuitsAtFirstFailedStage(t *testing.T) { + tests := []struct { + name string + failStage DoctorStage + wantCode ErrorCode + wantCalls []string + }{ + {"recovery", DoctorStageRecovery, ErrorRecoveryFailed, []string{"recovery"}}, + {"schema", DoctorStageSchema, ErrorOperationFailed, []string{"recovery", "schema"}}, + {"verify", DoctorStageVerify, ErrorVerificationFailed, []string{"recovery", "schema", "verify"}}, + {"audit", DoctorStageAudit, ErrorVerificationFailed, []string{"recovery", "schema", "verify", "audit"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + eng := newDoctorHookEngine(t) + var calls []string + eng.doctorRecover = func(context.Context) (RecoverResult, error) { + calls = append(calls, "recovery") + if tc.failStage == DoctorStageRecovery { + return RecoverResult{}, errors.New("recovery fault") + } + return RecoverResult{}, nil + } + eng.doctorSchema = func(*sql.DB) (int64, error) { + calls = append(calls, "schema") + if tc.failStage == DoctorStageSchema { + return 0, errors.New("schema fault") + } + return 8, nil + } + eng.doctorVerify = func(context.Context, string) error { + calls = append(calls, "verify") + if tc.failStage == DoctorStageVerify { + return errors.New("verify fault") + } + return nil + } + eng.doctorAudit = func(*sql.DB) (DoctorPhysicalAudit, DoctorSnapshotAudit, error) { + calls = append(calls, "audit") + if tc.failStage == DoctorStageAudit { + return DoctorPhysicalAudit{}, DoctorSnapshotAudit{}, errors.New("audit fault") + } + return DoctorPhysicalAudit{}, DoctorSnapshotAudit{}, nil + } + + result, err := eng.Doctor(context.Background(), DoctorRequest{}) + if err == nil || !IsCode(err, tc.wantCode) || !strings.Contains(err.Error(), "doctor "+tc.name) { + t.Fatalf("Doctor error=%v code=%q", err, CodeOf(err)) + } + if result.FailedStage != tc.failStage || !reflect.DeepEqual(calls, tc.wantCalls) { + t.Fatalf("result=%+v calls=%v want %v", result, calls, tc.wantCalls) + } + }) + } +} + +func TestDoctorRejectsInvalidLevelBeforeCorrectiveRecovery(t *testing.T) { + eng := newDoctorHookEngine(t) + called := false + eng.doctorRecover = func(context.Context) (RecoverResult, error) { + called = true + return RecoverResult{}, nil + } + _, err := eng.Doctor(context.Background(), DoctorRequest{VerifyLevel: "fast"}) + if !IsCode(err, ErrorInvalidArgument) || called { + t.Fatalf("invalid level error=%v code=%q recovery_called=%v", err, CodeOf(err), called) + } +} + +func newDoctorHookEngine(t *testing.T) *DefaultEngine { + t.Helper() + eng, err := New(Config{DB: newEngineTestDB(t), ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + return eng +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 02976b8d..0ae84041 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -103,6 +103,9 @@ type Engine interface { // Recover executes the corrective repository recovery pass against the // injected database and configured container directory. Recover(ctx context.Context, req RecoverRequest) (RecoverResult, error) + // Doctor executes recovery, schema validation, system verification, and + // integrity audit in order, stopping on the first failed stage. + Doctor(ctx context.Context, req DoctorRequest) (DoctorResult, error) // Remove removes logical files from the repository by logical file ID. // Safety invariant: Remove must never make valid data unrecoverable. diff --git a/internal/maintenance/verify_command.go b/internal/maintenance/verify_command.go index d7c49a01..5bbfba42 100644 --- a/internal/maintenance/verify_command.go +++ b/internal/maintenance/verify_command.go @@ -21,10 +21,12 @@ func CollectSystemAuditSummary() (SystemAuditSummary, error) { } defer func() { _ = dbconn.Close() }() - return collectSystemAuditSummaryWithDB(dbconn) + return CollectSystemAuditSummaryWithDB(dbconn) } -func collectSystemAuditSummaryWithDB(dbconn *sql.DB) (SystemAuditSummary, error) { +// CollectSystemAuditSummaryWithDB collects the Doctor audit summaries through +// a caller-owned database connection. +func CollectSystemAuditSummaryWithDB(dbconn *sql.DB) (SystemAuditSummary, error) { physical, err := verify.CheckPhysicalFileGraphIntegrity(dbconn) if err != nil { return SystemAuditSummary{}, err From 1828e10d5987755470ccbb62f494aaf2ae2e580c Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 16:02:45 +0200 Subject: [PATCH 19/27] Close application composition and thin CLI boundary --- cmd/coldkeep/application_session.go | 25 + cmd/coldkeep/legacy_application_seams_test.go | 322 ++++++++++ cmd/coldkeep/main.go | 562 +++++++----------- cmd/coldkeep/stored_path_engine_adapters.go | 82 +-- .../thin_cli_dependency_guard_test.go | 143 +++++ docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...cation-composition-and-thin-cli-closure.md | 86 +++ docs/release/v1.13/v1.13.12-scope.md | 6 +- .../v1.13/v1.13.12-validation-checklist.md | 19 +- internal/application/session.go | 98 +++ internal/application/session_test.go | 21 + internal/engine/candidates.go | 48 ++ internal/engine/candidates_test.go | 2 +- internal/engine/contracts_test.go | 4 + .../engine/default_engine_routing_helpers.go | 103 +++- internal/engine/dependency_guard_test.go | 10 +- internal/engine/engine.go | 4 + internal/engine/gc_engine_test.go | 26 + .../engine/gc_plan_backend_contract_test.go | 25 + 20 files changed, 1155 insertions(+), 439 deletions(-) create mode 100644 cmd/coldkeep/application_session.go create mode 100644 cmd/coldkeep/legacy_application_seams_test.go create mode 100644 cmd/coldkeep/thin_cli_dependency_guard_test.go create mode 100644 docs/release/v1.13/v1.13.12-phase18-application-composition-and-thin-cli-closure.md create mode 100644 internal/application/session.go create mode 100644 internal/application/session_test.go create mode 100644 internal/engine/gc_plan_backend_contract_test.go diff --git a/cmd/coldkeep/application_session.go b/cmd/coldkeep/application_session.go new file mode 100644 index 00000000..965fc88b --- /dev/null +++ b/cmd/coldkeep/application_session.go @@ -0,0 +1,25 @@ +package main + +import ( + "context" + "github.com/franchoy/coldkeep/internal/application" + "github.com/franchoy/coldkeep/internal/engine" +) + +type commandSession interface { + Engine() engine.Engine + OperationContext(context.Context) (context.Context, context.CancelFunc) + Close() error +} + +func openApplicationSession(req application.Request) (commandSession, error) { + return application.Open(req) +} + +var openApplicationSessionPhase = openApplicationSession + +func openCommandSession(operation string, requireStorage bool, containerDir string) (commandSession, error) { + return openApplicationSessionPhase(application.Request{ + Operation: operation, RequireStorage: requireStorage, ContainerDir: containerDir, + }) +} diff --git a/cmd/coldkeep/legacy_application_seams_test.go b/cmd/coldkeep/legacy_application_seams_test.go new file mode 100644 index 00000000..70272e25 --- /dev/null +++ b/cmd/coldkeep/legacy_application_seams_test.go @@ -0,0 +1,322 @@ +package main + +// This file contains the pre-v1.13.12 construction seams used by the frozen +// CLI compatibility suite. Production command code cannot see these symbols: +// normal builds compose dependencies only through internal/application. + +import ( + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "strings" + + "github.com/franchoy/coldkeep/internal/application" + "github.com/franchoy/coldkeep/internal/batch" + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/snapshot" + "github.com/franchoy/coldkeep/internal/storage" +) + +var loadDefaultStorageContextPhase = storage.LoadDefaultStorageContext +var createSnapshotPhase = snapshot.CreateSnapshotWithOptions +var restoreSnapshotPhase = snapshot.RestoreSnapshot +var deleteSnapshotPhase = snapshot.DeleteSnapshot +var snapshotDeleteLineagePreviewPhase = snapshot.LoadDeleteLineagePreview + +func loadSnapshotDeleteLineagePreview(ctx context.Context, dbconn *sql.DB, snapshotID string) (*snapshotDeleteLineagePreview, error) { + return snapshot.LoadDeleteLineagePreview(ctx, dbconn, snapshotID) +} + +var connectListSearchDBPhase = db.ConnectDB +var connectRepairDBPhase = db.ConnectDB +var connectRecoveryDBPhase = db.ConnectDB +var connectDoctorDBPhase = db.ConnectDB + +var newCommandEngine = func(dbconn *sql.DB, containerDir string) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn, ContainerDir: containerDir}) +} +var newSnapshotRestoreCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir(), StoreContext: &sgctx}) +} +var newStoreFolderCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir(), StoreContext: &sgctx}) +} +var newConfigurationCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} +var newSnapshotReadCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} +var newVerifyCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} +var newObservabilityCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { + return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) +} +var newRepairCommandEngine = func(dbconn *sql.DB) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn}) +} +var newRecoveryCommandEngine = func(dbconn *sql.DB, containersDir string) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn, ContainerDir: containersDir}) +} +var newDoctorCommandEngine = func(dbconn *sql.DB, containersDir string) (engine.Engine, error) { + return engine.New(engine.Config{DB: dbconn, ContainerDir: containersDir}) +} + +type legacyApplicationSession struct { + engine engine.Engine + close func() error +} + +func (s *legacyApplicationSession) Engine() engine.Engine { return s.engine } +func (s *legacyApplicationSession) OperationContext(parent context.Context) (context.Context, context.CancelFunc) { + return db.NewOperationContext(parent) +} +func (s *legacyApplicationSession) Close() error { + if s == nil || s.close == nil { + return nil + } + close := s.close + s.close = nil + return close() +} + +func init() { + openApplicationSessionPhase = openLegacyApplicationSession +} + +func openLegacyApplicationSession(req application.Request) (commandSession, error) { + if !req.RequireStorage { + return openLegacyDBSession(req) + } + sgctx, err := loadDefaultStorageContextPhase() + if err != nil { + return nil, err + } + var eng engine.Engine + switch req.Operation { + case "config": + eng, err = newConfigurationCommandEngine(sgctx) + case "store": + if reflect.ValueOf(storeByFilePhase).Pointer() != reflect.ValueOf(productionStoreByFilePhase).Pointer() { + eng = legacyPerItemEngine{Engine: nil, storage: &sgctx} + } else { + eng, err = newStoreFolderCommandEngine(sgctx) + } + case "store-folder": + eng, err = newStoreFolderCommandEngine(sgctx) + case "snapshot-list", "snapshot-show", "snapshot-stats", "snapshot-diff": + eng, err = newSnapshotReadCommandEngine(sgctx) + case "snapshot-restore": + eng, err = newSnapshotRestoreCommandEngine(sgctx) + case "verify": + eng, err = newVerifyCommandEngine(sgctx) + case "stats", "inspect", "simulate-gc": + eng, err = newObservabilityCommandEngine(sgctx) + default: + eng, err = newCommandEngine(sgctx.DB, effectiveLegacyContainerDir(sgctx, req.ContainerDir)) + if err == nil && ((req.Operation == "restore" && reflect.ValueOf(restoreByIDPhase).Pointer() != reflect.ValueOf(productionRestoreByIDPhase).Pointer()) || + (req.Operation == "remove" && reflect.ValueOf(removeByIDPhase).Pointer() != reflect.ValueOf(productionRemoveByIDPhase).Pointer())) { + eng = legacyPerItemEngine{Engine: eng, storage: &sgctx} + } + } + if err != nil { + _ = sgctx.Close() + return nil, err + } + return &legacyApplicationSession{engine: eng, close: sgctx.Close}, nil +} + +func openLegacyDBSession(req application.Request) (commandSession, error) { + connector := connectListSearchDBPhase + if req.Operation == "repair" { + connector = connectRepairDBPhase + } else if req.Operation == "recovery" { + connector = connectRecoveryDBPhase + } else if req.Operation == "doctor" { + connector = connectDoctorDBPhase + } + dbconn, err := connector() + if err != nil { + return nil, err + } + var eng engine.Engine + switch req.Operation { + case "repair": + eng, err = newRepairCommandEngine(dbconn) + case "recovery": + eng, err = newRecoveryCommandEngine(dbconn, req.ContainerDir) + case "doctor": + eng, err = newDoctorCommandEngine(dbconn, req.ContainerDir) + default: + eng, err = newCommandEngine(dbconn, req.ContainerDir) + } + if err != nil { + _ = dbconn.Close() + return nil, err + } + return &legacyApplicationSession{engine: eng, close: dbconn.Close}, nil +} + +func effectiveLegacyContainerDir(sgctx storage.StorageContext, override string) string { + if strings.TrimSpace(override) != "" { + return override + } + return sgctx.EffectiveContainerDir() +} + +var storeByFilePhase = func(sgctx *storage.StorageContext, path, codecName string) (storage.StoreFileResult, error) { + if sgctx == nil || sgctx.DB == nil { + return storage.StoreFileResult{}, fmt.Errorf("store: storage context DB is required") + } + eng, err := engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir(), StoreContext: sgctx}) + if err != nil { + return storage.StoreFileResult{}, err + } + res, err := eng.Store(context.Background(), engine.StoreRequest{SourcePath: path, Codec: strings.TrimSpace(codecName)}) + if err != nil { + return storage.StoreFileResult{}, err + } + return storage.StoreFileResult{FileID: res.LogicalFileID, FileHash: res.FileHash, Path: res.StoredPath, AlreadyStored: res.AlreadyStored}, nil +} + +var removeByIDPhase = func(sgctx *storage.StorageContext, fileID int64, dryRun bool) batch.ItemResult { + if sgctx == nil || sgctx.DB == nil { + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: "remove: storage context DB is required"} + } + eng, err := engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) + if err != nil { + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} + } + res, err := eng.Remove(context.Background(), engine.RemoveRequest{FileIDs: []int64{fileID}, DryRun: dryRun, FailFast: true}) + if err != nil || len(res.Items) != 1 { + if err == nil { + err = fmt.Errorf("remove: expected one item result, got %d", len(res.Items)) + } + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} + } + item := res.Items[0] + if item.Status == engine.BatchItemFailed { + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: item.Error, InvariantCode: item.InvariantCode, RecommendedAction: item.RecommendedAction} + } + if dryRun { + return batch.ItemResult{ID: fileID, Status: batch.ResultPlanned, Message: "would remove"} + } + return batch.ItemResult{ID: fileID, Status: batch.ResultSuccess, Message: fmt.Sprintf("removed mappings=%d", item.RemovedChunkAssociations)} +} + +var restoreByIDPhase = func(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool, dryRun bool) (storage.RestoreFileResult, error) { + if sgctx == nil || sgctx.DB == nil { + return storage.RestoreFileResult{}, fmt.Errorf("restore: storage context DB is required") + } + eng, err := engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) + if err != nil { + return storage.RestoreFileResult{}, err + } + res, err := eng.Restore(context.Background(), engine.RestoreRequest{FileIDs: []int64{fileID}, DestinationRoot: outputDir, Overwrite: overwrite, DryRun: dryRun, FailFast: true}) + if err != nil || len(res.Items) != 1 { + if err == nil { + err = fmt.Errorf("restore: expected one item result, got %d", len(res.Items)) + } + return storage.RestoreFileResult{}, err + } + item := res.Items[0] + if item.Status == engine.BatchItemFailed { + return storage.RestoreFileResult{}, errors.New(item.Error) + } + return storage.RestoreFileResult{FileID: fileID, OriginalName: item.OriginalName, OutputPath: item.DestinationPath, RestoredHash: item.RestoredHash}, nil +} + +var productionStoreByFilePhase = storeByFilePhase +var productionRemoveByIDPhase = removeByIDPhase +var productionRestoreByIDPhase = restoreByIDPhase + +type legacyPerItemEngine struct { + engine.Engine + storage *storage.StorageContext +} + +func (e legacyPerItemEngine) Store(_ context.Context, req engine.StoreRequest) (engine.StoreResult, error) { + result, err := storeByFilePhase(e.storage, req.SourcePath, req.Codec) + return engine.StoreResult{ + LogicalFileID: result.FileID, FileHash: result.FileHash, + StoredPath: result.Path, AlreadyStored: result.AlreadyStored, + }, err +} + +func (e legacyPerItemEngine) Remove(_ context.Context, req engine.RemoveRequest) (engine.RemoveResult, error) { + result := engine.RemoveResult{DryRun: req.DryRun} + for _, fileID := range req.FileIDs { + legacy := removeByIDPhase(e.storage, fileID, req.DryRun) + item := engine.RemoveItemResult{FileID: fileID} + switch legacy.Status { + case batch.ResultSuccess: + item.Status = engine.BatchItemOK + _, _ = fmt.Sscanf(legacy.Message, "removed mappings=%d", &item.RemovedChunkAssociations) + case batch.ResultPlanned: + item.Status = engine.BatchItemOK + default: + item.Status, item.Error = engine.BatchItemFailed, legacy.Message + item.InvariantCode, item.RecommendedAction = legacy.InvariantCode, legacy.RecommendedAction + } + result.Items = append(result.Items, item) + if item.Status == engine.BatchItemFailed { + result.Summary.Failed++ + if req.FailFast { + break + } + } else { + result.Summary.OK++ + } + } + return result, nil +} + +func (e legacyPerItemEngine) Restore(_ context.Context, req engine.RestoreRequest) (engine.RestoreResult, error) { + result := engine.RestoreResult{DryRun: req.DryRun} + for _, fileID := range req.FileIDs { + legacy, err := restoreByIDPhase(e.storage, fileID, req.DestinationRoot, req.Overwrite, req.DryRun) + item := engine.RestoreItemResult{ + FileID: fileID, OriginalName: legacy.OriginalName, + DestinationPath: legacy.OutputPath, RestoredHash: legacy.RestoredHash, + Status: engine.BatchItemOK, + } + if err != nil { + item.Status, item.Error = engine.BatchItemFailed, err.Error() + } + result.Items = append(result.Items, item) + if item.Status == engine.BatchItemFailed { + result.Summary.Failed++ + if req.FailFast { + break + } + } else { + result.Summary.OK++ + } + } + return result, nil +} + +func executeRestoreDryRunItem(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool) batch.ItemResult { + result, err := restoreByIDPhase(sgctx, fileID, outputDir, overwrite, true) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: fmt.Sprintf("file ID %d not found", fileID)} + } + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} + } + return batch.ItemResult{ID: fileID, Status: batch.ResultPlanned, Message: fmt.Sprintf("would restore -> %s", result.OutputPath), OriginalName: result.OriginalName, OutputPath: result.OutputPath} +} + +func executeRestoreItem(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool) batch.ItemResult { + result, err := restoreByIDPhase(sgctx, fileID, outputDir, overwrite, false) + if err != nil { + item := batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} + annotateBatchFailureFromError(err, &item) + return item + } + return batch.ItemResult{ID: fileID, Status: batch.ResultSuccess, Message: "restored", OriginalName: result.OriginalName, OutputPath: result.OutputPath} +} diff --git a/cmd/coldkeep/main.go b/cmd/coldkeep/main.go index adcde1e2..1dca328a 100644 --- a/cmd/coldkeep/main.go +++ b/cmd/coldkeep/main.go @@ -186,131 +186,14 @@ const doctorDefaultVerifyLevel = verify.VerifyStandard const doctorOperationalHint = "After significant operations, run coldkeep doctor to validate system health." -var storeByFilePhase = func(sgctx *storage.StorageContext, path, codecName string) (storage.StoreFileResult, error) { - if sgctx == nil || sgctx.DB == nil { - return storage.StoreFileResult{}, fmt.Errorf("store: storage context DB is required") - } - eng, err := engine.New(engine.Config{ - DB: sgctx.DB, - ContainerDir: sgctx.EffectiveContainerDir(), - StoreContext: sgctx, - }) - if err != nil { - return storage.StoreFileResult{}, err - } - - res, err := eng.Store(context.Background(), engine.StoreRequest{ - SourcePath: path, - Codec: strings.TrimSpace(codecName), - }) - if err != nil { - return storage.StoreFileResult{}, err - } - - return storage.StoreFileResult{ - FileID: res.LogicalFileID, - FileHash: res.FileHash, - Path: res.StoredPath, - AlreadyStored: res.AlreadyStored, - }, nil -} -var removeByIDPhase = func(sgctx *storage.StorageContext, fileID int64, dryRun bool) batch.ItemResult { - // By-ID remove remains the active engine-owned path. - if sgctx == nil || sgctx.DB == nil { - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: "remove: storage context DB is required"} - } - - eng, err := engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) - if err != nil { - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} - } - - res, err := eng.Remove(context.Background(), engine.RemoveRequest{ - FileIDs: []int64{fileID}, - DryRun: dryRun, - FailFast: true, - }) - if err != nil { - item := batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} - annotateBatchFailureFromError(err, &item) - return item - } - if len(res.Items) != 1 { - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: fmt.Sprintf("remove: expected one item result, got %d", len(res.Items))} - } - - item := res.Items[0] - if item.Status == engine.BatchItemFailed { - return batch.ItemResult{ - ID: fileID, - Status: batch.ResultFailed, - Message: item.Error, - InvariantCode: item.InvariantCode, - RecommendedAction: item.RecommendedAction, - } - } - - if dryRun { - return batch.ItemResult{ID: fileID, Status: batch.ResultPlanned, Message: "would remove"} - } - return batch.ItemResult{ID: fileID, Status: batch.ResultSuccess, Message: fmt.Sprintf("removed mappings=%d", item.RemovedChunkAssociations)} -} -var restoreByIDPhase = func(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool, dryRun bool) (storage.RestoreFileResult, error) { - // By-ID restore remains the active engine-owned path. - if sgctx == nil || sgctx.DB == nil { - return storage.RestoreFileResult{}, fmt.Errorf("restore: storage context DB is required") - } - eng, err := engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) - if err != nil { - return storage.RestoreFileResult{}, err - } - - info, err := storage.GetLogicalFileInfoWithDB(sgctx.DB, fileID) - if err != nil { - return storage.RestoreFileResult{}, err - } - - res, err := eng.Restore(context.Background(), engine.RestoreRequest{ - FileIDs: []int64{fileID}, - DestinationRoot: outputDir, - Overwrite: overwrite, - DryRun: dryRun, - FailFast: true, - }) - if err != nil { - return storage.RestoreFileResult{}, err - } - if len(res.Items) != 1 { - return storage.RestoreFileResult{}, fmt.Errorf("restore: expected one item result, got %d", len(res.Items)) - } - item := res.Items[0] - if item.Status == engine.BatchItemFailed { - return storage.RestoreFileResult{}, errors.New(item.Error) - } - - return storage.RestoreFileResult{ - FileID: fileID, - OriginalName: info.OriginalName, - OutputPath: item.DestinationPath, - RestoredHash: item.RestoredHash, - }, nil -} var runGCPhase = func(dryRun bool, containersDir string) (maintenance.GCResult, error) { - sgctx, err := loadDefaultStorageContextPhase() - if err != nil { - return maintenance.GCResult{}, err - } - defer func() { _ = sgctx.Close() }() - - eng, err := engine.New(engine.Config{ - DB: sgctx.DB, - ContainerDir: containersDir, - }) + session, err := openCommandSession("gc", true, containersDir) if err != nil { return maintenance.GCResult{}, err } + defer func() { _ = session.Close() }() - result, err := eng.GarbageCollect(context.Background(), engine.GarbageCollectRequest{DryRun: dryRun}) + result, err := session.Engine().GarbageCollect(context.Background(), engine.GarbageCollectRequest{DryRun: dryRun}) if err != nil { return maintenance.GCResult{}, err } @@ -326,25 +209,9 @@ var runGCPhase = func(dryRun bool, containersDir string) (maintenance.GCResult, }, nil } var startupRecoveryPhase = runRecoveryThroughEngine -var loadDefaultStorageContextPhase = storage.LoadDefaultStorageContext -// Compatibility-only direct snapshot-domain create seam retained for lower- -// level tests and non-CLI callers. Production CLI snapshot create routes -// through Engine.SnapshotCreate. -var createSnapshotPhase = snapshot.CreateSnapshotWithOptions - -// Compatibility-only direct snapshot-domain restore seam retained for lower- -// level tests and non-CLI callers. Production CLI snapshot restore routes -// through Engine.SnapshotRestore. -var restoreSnapshotPhase = snapshot.RestoreSnapshot var currentWorkingDirectoryPhase = os.Getwd -// Compatibility-only direct snapshot-domain delete seams remain for lower- -// level tests and non-CLI callers. Production CLI snapshot delete routes -// through Engine.SnapshotDelete. -var deleteSnapshotPhase = snapshot.DeleteSnapshot -var snapshotDeleteLineagePreviewPhase = loadSnapshotDeleteLineagePreview - // snapshotMetaToSnapshot maps an engine.SnapshotMeta to a snapshot.Snapshot for // CLI renderers that expect the snapshot package's type with sql.NullString fields. func snapshotMetaToSnapshot(m engine.SnapshotMeta) snapshot.Snapshot { @@ -359,18 +226,13 @@ func snapshotMetaToSnapshot(m engine.SnapshotMeta) snapshot.Snapshot { } var runObservabilityStatsPhase = func(opts observability.StatsOptions) (*observability.StatsResult, error) { - sgctx, err := loadDefaultStorageContextPhase() - if err != nil { - return nil, err - } - defer func() { _ = sgctx.DB.Close() }() - - eng, err := newObservabilityCommandEngine(sgctx) + session, err := openCommandSession("stats", true, "") if err != nil { return nil, err } + defer func() { _ = session.Close() }() - result, err := eng.Stats(context.Background(), engine.StatsRequest{ + result, err := session.Engine().Stats(context.Background(), engine.StatsRequest{ IncludeContainers: opts.IncludeContainers, IncludeTrace: opts.Trace.Enabled, }) @@ -384,17 +246,12 @@ var runObservabilityStatsPhase = func(opts observability.StatsOptions) (*observa return statsResultFromEngine(result), nil } var runObservabilityInspectPhase = func(entity observability.EntityType, id string, opts observability.InspectOptions) (*observability.InspectResult, error) { - sgctx, err := loadDefaultStorageContextPhase() - if err != nil { - return nil, err - } - defer func() { _ = sgctx.Close() }() - - eng, err := newObservabilityCommandEngine(sgctx) + session, err := openCommandSession("inspect", true, "") if err != nil { return nil, err } - result, err := eng.Inspect(context.Background(), engine.InspectRequest{ + defer func() { _ = session.Close() }() + result, err := session.Engine().Inspect(context.Background(), engine.InspectRequest{ Entity: engine.InspectEntity(entity), EntityID: id, Options: engine.InspectOptions{ Deep: opts.Deep, Relations: opts.Relations, Reverse: opts.Reverse, @@ -413,7 +270,11 @@ var runObservabilityInspectPhase = func(entity observability.EntityType, id stri // Simulation and benchmark commands are explicitly classified tooling. Their // isolated simulated repositories retain the observability service seam; no // production inspect, stats, or verify command uses it. -var newObservabilityServicePhase = observability.NewService +func newSimulationObservabilityService(dbconn *sql.DB) (*observability.Service, error) { + return observability.NewService(dbconn) +} + +var newObservabilityServicePhase = newSimulationObservabilityService var runChunkerBenchmarkPhase = runChunkerBenchmark var runCoreBenchmarkPhase = runCoreBenchmark var runBenchmarkDeterminismPhase = validateBenchmarkDeterminism @@ -1055,15 +916,12 @@ func runConfigCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return usageErrorf("unknown config subcommand: %s", parsed.positionals[0]) } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("config", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() - eng, err := newConfigurationCommandEngine(sgctx) - if err != nil { - return err - } + defer func() { _ = session.Close() }() + eng := session.Engine() engineKey := engine.ConfigurationKey(key) switch subcommand { @@ -1364,16 +1222,16 @@ func runStoreCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { codecName, _ := parsed.lastFlagValue("codec") perf := newPerfTimer() - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("store", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() perf.Mark("setup") - var result storage.StoreFileResult + var engineResult engine.StoreResult if codecName == "" { - result, err = storeByFilePhase(&sgctx, path, "") + engineResult, err = session.Engine().Store(context.Background(), engine.StoreRequest{SourcePath: path}) } else { if codecName == "plain" { _, _ = fmt.Fprintln(os.Stderr, "WARNING: data would be stored without encryption") @@ -1383,16 +1241,18 @@ func runStoreCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { if parseErr != nil { return parseErr } - result, err = storeByFilePhase(&sgctx, path, string(codec)) + engineResult, err = session.Engine().Store(context.Background(), engine.StoreRequest{SourcePath: path, Codec: string(codec)}) } perf.Mark("operation") - if sgctx.Writer != nil { - _ = sgctx.Writer.FinalizeContainer() - } + _ = session.Close() perf.Mark("finalize") if err != nil { return err } + result := storage.StoreFileResult{ + FileID: engineResult.LogicalFileID, FileHash: engineResult.FileHash, + Path: engineResult.StoredPath, AlreadyStored: engineResult.AlreadyStored, + } if outputMode == outputModeJSON { payload := map[string]any{ @@ -1445,20 +1305,16 @@ func runStoreFolderCommand(parsed parsedCommandLine, outputMode cliOutputMode) e opts.StoreFolderWorkers = workers } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("store-folder", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() if codecName == "plain" { _, _ = fmt.Fprintln(os.Stderr, "WARNING: data would be stored without encryption") } - eng, err := newStoreFolderCommandEngine(sgctx) - if err != nil { - return err - } - _, err = eng.StoreFolder(context.Background(), engine.StoreFolderRequest{ + _, err = session.Engine().StoreFolder(context.Background(), engine.StoreFolderRequest{ SourcePath: path, Codec: codecName, Workers: opts.StoreFolderWorkers, @@ -1526,20 +1382,16 @@ func runRestoreCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } perf := newPerfTimer() - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("restore-stored-path", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() perf.Mark("setup") - eng, err := newCommandEngine(sgctx.DB, sgctx.EffectiveContainerDir()) - if err != nil { - return err - } result, normalizedMode, err := restoreStoredPathWithEngine( context.Background(), - eng, + session.Engine(), storedPath, destinationMode, destination, @@ -1622,21 +1474,22 @@ func runRestoreCommand(parsed parsedCommandLine, outputMode cliOutputMode) error } restorePerf := newPerfTimer() - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("restore", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() restorePerf.Mark("setup") - execFunc := func(fileID int64) batch.ItemResult { - if dryRun { - return executeRestoreDryRunItem(&sgctx, fileID, outputPath, overwrite) - } - return executeRestoreItem(&sgctx, fileID, outputPath, overwrite) + fileIDs := executablePreparedIDs(preparedTargets) + result, err := session.Engine().Restore(context.Background(), engine.RestoreRequest{ + FileIDs: fileIDs, DestinationRoot: outputPath, Overwrite: overwrite, + DryRun: dryRun, FailFast: failFast, + }) + if err != nil { + return err } - - report := batch.ExecutePrepared(batch.OperationRestore, dryRun, failFast, preparedTargets, execFunc) + report := preparedRestoreReport(preparedTargets, result, dryRun, failFast) restorePerf.Mark("operation") return emitBatchCommandReport("restore", report, outputMode, restorePerf.Spans()) } @@ -1686,17 +1539,13 @@ func runRemoveCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return usageErrorf("--dry-run and --fail-fast are not supported with --stored-path") } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("remove-stored-path", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() - eng, err := newCommandEngine(sgctx.DB, sgctx.EffectiveContainerDir()) - if err != nil { - return err - } - result, err := removeStoredPathWithEngine(context.Background(), eng, storedPath) + result, err := removeStoredPathWithEngine(context.Background(), session.Engine(), storedPath) if err != nil { return err } @@ -1760,18 +1609,14 @@ func runRemoveCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return emitBatchCommandReport("remove", report, outputMode, removePerf.Spans()) } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("remove-stored-paths", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() removePerf.Mark("setup") - eng, err := newCommandEngine(sgctx.DB, sgctx.EffectiveContainerDir()) - if err != nil { - return err - } - result, err := eng.RemoveStoredPaths(context.Background(), req) + result, err := session.Engine().RemoveStoredPaths(context.Background(), req) if err != nil { return err } @@ -1805,18 +1650,20 @@ func runRemoveCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return emitBatchCommandReport("remove", report, outputMode, removePerf.Spans()) } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("remove", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() removePerf.Mark("setup") - execFunc := func(fileID int64) batch.ItemResult { - return removeByIDPhase(&sgctx, fileID, dryRun) + result, err := session.Engine().Remove(context.Background(), engine.RemoveRequest{ + FileIDs: executablePreparedIDs(preparedTargets), DryRun: dryRun, FailFast: failFast, + }) + if err != nil { + return err } - - report := batch.ExecutePrepared(batch.OperationRemove, dryRun, failFast, preparedTargets, execFunc) + report := preparedRemoveReport(preparedTargets, result, dryRun, failFast) removePerf.Mark("operation") return emitBatchCommandReport("remove", report, outputMode, removePerf.Spans()) } @@ -1921,39 +1768,64 @@ func printBatchHumanReport(label string, report batch.Report) { } } -func executeRestoreDryRunItem(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool) batch.ItemResult { - result, err := restoreByIDPhase(sgctx, fileID, outputDir, overwrite, true) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: fmt.Sprintf("file ID %d not found", fileID)} +func executablePreparedIDs(targets []batch.PreparedTarget) []int64 { + ids := make([]int64, 0, len(targets)) + for _, target := range targets { + if target.Executable { + ids = append(ids, target.ID) } - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} - } - - return batch.ItemResult{ - ID: fileID, - Status: batch.ResultPlanned, - Message: fmt.Sprintf("would restore -> %s", result.OutputPath), - OriginalName: result.OriginalName, - OutputPath: result.OutputPath, } + return ids } -func executeRestoreItem(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool) batch.ItemResult { - result, err := restoreByIDPhase(sgctx, fileID, outputDir, overwrite, false) - if err != nil { - item := batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} - annotateBatchFailureFromError(err, &item) - return item - } +func preparedRestoreReport(targets []batch.PreparedTarget, result engine.RestoreResult, dryRun, failFast bool) batch.Report { + index := 0 + return batch.ExecutePrepared(batch.OperationRestore, dryRun, failFast, targets, func(fileID int64) batch.ItemResult { + if index >= len(result.Items) { + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: "restore engine omitted item result"} + } + item := result.Items[index] + index++ + if item.Status == engine.BatchItemFailed { + failure := batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: item.Error} + annotateBatchFailureFromError(errors.New(item.Error), &failure) + return failure + } + status := batch.ResultSuccess + message := "restored" + if dryRun { + status = batch.ResultPlanned + message = fmt.Sprintf("would restore -> %s", item.DestinationPath) + } + return batch.ItemResult{ + ID: fileID, Status: status, Message: message, + OriginalName: item.OriginalName, OutputPath: item.DestinationPath, + } + }) +} - return batch.ItemResult{ - ID: fileID, - Status: batch.ResultSuccess, - Message: "restored", - OriginalName: result.OriginalName, - OutputPath: result.OutputPath, - } +func preparedRemoveReport(targets []batch.PreparedTarget, result engine.RemoveResult, dryRun, failFast bool) batch.Report { + index := 0 + return batch.ExecutePrepared(batch.OperationRemove, dryRun, failFast, targets, func(fileID int64) batch.ItemResult { + if index >= len(result.Items) { + return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: "remove engine omitted item result"} + } + item := result.Items[index] + index++ + if item.Status == engine.BatchItemFailed { + return batch.ItemResult{ + ID: fileID, Status: batch.ResultFailed, Message: item.Error, + InvariantCode: item.InvariantCode, RecommendedAction: item.RecommendedAction, + } + } + if dryRun { + return batch.ItemResult{ID: fileID, Status: batch.ResultPlanned, Message: "would remove"} + } + return batch.ItemResult{ + ID: fileID, Status: batch.ResultSuccess, + Message: fmt.Sprintf("removed mappings=%d", item.RemovedChunkAssociations), + } + }) } func annotateBatchFailureFromError(err error, item *batch.ItemResult) { @@ -2368,18 +2240,14 @@ func rawRepairTargetValues(raw []batch.RawTarget) []string { } func executeRepairEngine(targets []string, failFast bool) (engine.RepairResult, error) { - dbconn, err := connectRepairDBPhase() + session, err := openCommandSession("repair", false, "") if err != nil { return engine.RepairResult{}, fmt.Errorf("connect repair database: %w", err) } - defer func() { _ = dbconn.Close() }() - eng, err := newRepairCommandEngine(dbconn) - if err != nil { - return engine.RepairResult{}, err - } - ctx, cancel := db.NewOperationContext(context.Background()) + defer func() { _ = session.Close() }() + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - return eng.Repair(ctx, engine.RepairRequest{Targets: append([]string(nil), targets...), FailFast: failFast}) + return session.Engine().Repair(ctx, engine.RepairRequest{Targets: append([]string(nil), targets...), FailFast: failFast}) } func executeSingleRepairEngine(target string) (engine.RepairTargetResult, error) { @@ -2441,16 +2309,12 @@ func runListCommand(parsed parsedCommandLine, outputMode cliOutputMode) error { if err != nil { return err } - dbconn, err := connectListSearchDBPhase() + session, err := openCommandSession("list", false, "") if err != nil { return fmt.Errorf("failed to connect to DB: %w", err) } - defer func() { _ = dbconn.Close() }() - eng, err := newCommandEngine(dbconn, "") - if err != nil { - return err - } - result, err := eng.ListFiles(context.Background(), req) + defer func() { _ = session.Close() }() + result, err := session.Engine().ListFiles(context.Background(), req) if err != nil { return err } @@ -2498,16 +2362,12 @@ func runSearchCommand(parsed parsedCommandLine, outputMode cliOutputMode) error if err != nil { return err } - dbconn, err := connectListSearchDBPhase() + session, err := openCommandSession("search", false, "") if err != nil { return fmt.Errorf("failed to connect to DB: %w", err) } - defer func() { _ = dbconn.Close() }() - eng, err := newCommandEngine(dbconn, "") - if err != nil { - return err - } - result, err := eng.SearchFiles(context.Background(), req) + defer func() { _ = session.Close() }() + result, err := session.Engine().SearchFiles(context.Background(), req) if err != nil { return err } @@ -2586,18 +2446,14 @@ func runVerifyCommand(parsed parsedCommandLine, outputMode cliOutputMode) error return usageErrorf("Unknown target for verify: %s (expected 'system' or 'file ')", target) } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("verify", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() - eng, err := newVerifyCommandEngine(sgctx) - if err != nil { - return err - } - ctx, cancel := db.NewOperationContext(context.Background()) + defer func() { _ = session.Close() }() + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - result, err := eng.Verify(ctx, engine.VerifyRequest{ + result, err := session.Engine().Verify(ctx, engine.VerifyRequest{ Target: target, FileID: fileID, Level: verifyLevelToString(verifyLevel), }) if err != nil { @@ -4875,18 +4731,71 @@ func resolveRenderer(outputMode cliOutputMode) clirender.Renderer { } var runObservabilitySimulateGCPhase = func(opts observability.SimulationOptions) (*observability.SimulationResult, error) { - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("simulate-gc", true, "") if err != nil { return nil, err } - defer func() { _ = sgctx.DB.Close() }() - - svc, err := newObservabilityServicePhase(sgctx.DB) + defer func() { _ = session.Close() }() + result, err := session.Engine().PlanGarbageCollection(context.Background(), engine.GarbageCollectionPlanRequest{ + SnapshotIDsToOmit: append([]string(nil), opts.AssumeDeletedSnapshots...), + IncludeTrace: opts.Trace.Enabled, + }) + if traceErr := replayEngineTrace(opts.Trace, result.Trace); traceErr != nil { + return nil, traceErr + } if err != nil { return nil, err } + return gcSimulationFromEngine(result, time.Now().UTC()), nil +} - return svc.Simulate(context.Background(), opts) +func gcSimulationFromEngine(result engine.GarbageCollectionPlanResult, generatedAt time.Time) *observability.SimulationResult { + summary := result.Summary + impacts := make([]observability.ContainerSimulationImpact, len(result.Containers)) + for i, item := range result.Containers { + impacts[i] = observability.ContainerSimulationImpact{ + ContainerID: item.ContainerID, Filename: item.Filename, TotalBytes: item.TotalBytes, + LiveBytesAfterGC: item.LiveBytesAfterGC, ReclaimableBytes: item.ReclaimableBytes, + ReclaimableChunks: item.ReclaimableChunks, TotalChunks: item.TotalChunks, + FullyReclaimable: item.FullyReclaimable, RequiresCompaction: item.RequiresCompaction, + } + } + warnings := make([]observability.ObservationWarning, len(result.Warnings)) + for i, warning := range result.Warnings { + warnings[i] = observability.ObservationWarning{Code: warning.Code, Message: warning.Message} + } + gcResult := &observability.GCSimulationResult{ + GeneratedAtUTC: generatedAt, Kind: observability.SimulationKindGC, Exact: true, + Assumptions: observability.GCSimulationAssumptions{DeletedSnapshots: append([]string(nil), result.SnapshotIDsToOmit...)}, + Summary: observability.GCSimulationSummary{ + ReachableChunks: summary.ReachableChunks, UnreachableChunks: summary.UnreachableChunks, + LogicallyReclaimableBytes: summary.LogicallyReclaimableBytes, + PhysicallyReclaimableBytes: summary.PhysicallyReclaimableBytes, + FullyReclaimableContainers: summary.FullyReclaimableContainers, + PartiallyDeadContainers: summary.PartiallyDeadContainers, + PackedBlocksLive: summary.PackedBlocksLive, PackedBlocksDead: summary.PackedBlocksDead, + PackedBytesLive: summary.PackedBytesLive, PackedBytesReclaimable: summary.PackedBytesReclaimable, + RetainedDeadBytesDueToPackedBlocks: summary.RetainedDeadBytesDueToPackedBlocks, + }, + Containers: impacts, Warnings: warnings, + } + return &observability.SimulationResult{ + GeneratedAtUTC: generatedAt, Kind: observability.SimulationKindGC, Exact: true, + Summary: map[string]any{ + "total_chunks": summary.TotalChunks, "reachable_chunks": summary.ReachableChunks, + "unreachable_chunks": summary.UnreachableChunks, + "logically_reclaimable_bytes": summary.LogicallyReclaimableBytes, + "physically_reclaimable_bytes": summary.PhysicallyReclaimableBytes, + "fully_reclaimable_containers": summary.FullyReclaimableContainers, + "partially_dead_containers": summary.PartiallyDeadContainers, + "packed_blocks_live": summary.PackedBlocksLive, "packed_blocks_dead": summary.PackedBlocksDead, + "packed_bytes_live": summary.PackedBytesLive, + "packed_bytes_reclaimable": summary.PackedBytesReclaimable, + "retained_dead_bytes_due_to_packed_blocks": summary.RetainedDeadBytesDueToPackedBlocks, + "affected_containers": len(result.Containers), + }, + GC: gcResult, Warnings: warnings, + } } var runObservabilitySimulateStoreReportPhase = func(dbconn *sql.DB, subcommand, path string) (*observability.SimulateStoreReport, error) { @@ -4988,16 +4897,12 @@ func parseSnapshotDateFlag(flagName, value string, endOfDay bool) (*time.Time, e return &parsedDate, nil } -func loadSnapshotDB() (storage.StorageContext, error) { - sgctx, err := loadDefaultStorageContextPhase() +func loadSnapshotSession(operation string) (commandSession, error) { + session, err := openCommandSession(operation, true, "") if err != nil { - return storage.StorageContext{}, fmt.Errorf("load storage context: %w", err) + return nil, fmt.Errorf("load storage context: %w", err) } - if sgctx.DB == nil { - _ = sgctx.Close() - return storage.StorageContext{}, errors.New("storage context DB is nil") - } - return sgctx, nil + return session, nil } // parseSnapshotQuery builds a SnapshotQuery from the query-related flags in parsed. @@ -5486,19 +5391,16 @@ func runSnapshotListCommand(parsed parsedCommandLine, outputMode cliOutputMode) filter.Limit = parsedLimit } - sgctx, err := loadSnapshotDB() + session, err := loadSnapshotSession("snapshot-list") if err != nil { return err } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - eng, err := newSnapshotReadCommandEngine(sgctx) - if err != nil { - return err - } + eng := session.Engine() req := engine.SnapshotListRequest{ Since: filter.Since, Until: filter.Until, Limit: filter.Limit, Tree: treeMode, } @@ -5583,19 +5485,16 @@ func runSnapshotShowCommand(parsed parsedCommandLine, outputMode cliOutputMode) return err } - sgctx, err := loadSnapshotDB() + session, err := loadSnapshotSession("snapshot-show") if err != nil { return err } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - eng, err := newSnapshotReadCommandEngine(sgctx) - if err != nil { - return err - } + eng := session.Engine() result, err := eng.SnapshotShow(ctx, engine.SnapshotShowRequest{ SnapshotID: snapshotID, Query: snapshotQueryToEngine(query, limit), @@ -5658,19 +5557,16 @@ func runSnapshotStatsCommand(parsed parsedCommandLine, outputMode cliOutputMode) } } - sgctx, err := loadSnapshotDB() + session, err := loadSnapshotSession("snapshot-stats") if err != nil { return err } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - eng, err := newSnapshotReadCommandEngine(sgctx) - if err != nil { - return err - } + eng := session.Engine() result, err := eng.SnapshotStats(ctx, engine.SnapshotStatsRequest{SnapshotID: snapshotID}) if err != nil { return err @@ -5749,16 +5645,16 @@ func runSnapshotDeleteCommand(parsed parsedCommandLine, outputMode cliOutputMode return err } - sgctx, err := loadSnapshotDB() + session, err := loadSnapshotSession("snapshot-delete") if err != nil { return err } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - result, err := runSnapshotDeleteEngine(ctx, sgctx, req) + result, err := runSnapshotDeleteEngine(ctx, session.Engine(), req) if err != nil { return err } @@ -5811,13 +5707,9 @@ func parseSnapshotDeleteCommandRequest(parsed parsedCommandLine) (snapshotDelete func runSnapshotDeleteEngine( ctx context.Context, - sgctx storage.StorageContext, + eng engine.Engine, req snapshotDeleteCommandRequest, ) (engine.SnapshotDeleteResult, error) { - eng, err := newCommandEngine(sgctx.DB, sgctx.EffectiveContainerDir()) - if err != nil { - return engine.SnapshotDeleteResult{}, err - } return eng.SnapshotDelete(ctx, engine.SnapshotDeleteRequest{ SnapshotID: req.SnapshotID, Mode: req.Mode, @@ -5905,10 +5797,6 @@ func snapshotDeletePreviewFromEngineResult(result engine.SnapshotDeleteResult) ( return preview, nil } -func loadSnapshotDeleteLineagePreview(ctx context.Context, dbconn *sql.DB, snapshotID string) (*snapshotDeleteLineagePreview, error) { - return snapshot.LoadDeleteLineagePreview(ctx, dbconn, snapshotID) -} - func previewParentID(preview *snapshotDeleteLineagePreview) sql.NullString { if preview == nil { return sql.NullString{} @@ -6016,19 +5904,16 @@ func runSnapshotDiffCommand(parsed parsedCommandLine, outputMode cliOutputMode) return err } - sgctx, err := loadSnapshotDB() + session, err := loadSnapshotSession("snapshot-diff") if err != nil { return err } - defer func() { _ = sgctx.Close() }() + defer func() { _ = session.Close() }() - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - eng, err := newSnapshotReadCommandEngine(sgctx) - if err != nil { - return err - } + eng := session.Engine() engineResult, err := eng.SnapshotDiff(ctx, engine.SnapshotDiffRequest{ BaseID: baseID, TargetID: targetID, Summary: summaryMode, Filter: engine.SnapshotDiffFilter(filterType), Query: snapshotQueryToEngine(query, 0), @@ -6133,22 +6018,15 @@ func runSnapshotCreateCommand(parsed parsedCommandLine, outputMode cliOutputMode parentID = trimmed } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("snapshot-create", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() - - if sgctx.DB == nil { - return errors.New("storage context DB is nil") - } - eng, err := newCommandEngine(sgctx.DB, sgctx.EffectiveContainerDir()) - if err != nil { - return err - } + defer func() { _ = session.Close() }() + eng := session.Engine() perf.Mark("setup") - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() result, err := eng.SnapshotCreate(ctx, engine.SnapshotCreateRequest{ @@ -6248,17 +6126,13 @@ func runSnapshotRestoreCommand(parsed parsedCommandLine, outputMode cliOutputMod return err } - sgctx, err := loadDefaultStorageContextPhase() + session, err := openCommandSession("snapshot-restore", true, "") if err != nil { return fmt.Errorf("load storage context: %w", err) } - defer func() { _ = sgctx.Close() }() - - if sgctx.DB == nil { - return errors.New("storage context DB is nil") - } + defer func() { _ = session.Close() }() - ctx, cancel := db.NewOperationContext(context.Background()) + ctx, cancel := session.OperationContext(context.Background()) defer cancel() req, jsonOutputRoot, err := buildSnapshotRestoreEngineRequest(snapshotID, paths, selection, destinationMode, destination, parsed) @@ -6266,7 +6140,7 @@ func runSnapshotRestoreCommand(parsed parsedCommandLine, outputMode cliOutputMod return err } - result, err := runSnapshotRestoreEngine(ctx, sgctx, req) + result, err := runSnapshotRestoreEngine(ctx, session.Engine(), req) if err != nil { return err } @@ -6548,13 +6422,9 @@ func buildSnapshotRestoreEngineRequest( func runSnapshotRestoreEngine( ctx context.Context, - sgctx storage.StorageContext, + eng engine.Engine, req snapshotRestoreCommandRequest, ) (engine.SnapshotRestoreResult, error) { - eng, err := newSnapshotRestoreCommandEngine(sgctx) - if err != nil { - return engine.SnapshotRestoreResult{}, err - } return eng.SnapshotRestore(ctx, req.EngineRequest) } diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index 3718d2c0..352812fc 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -2,82 +2,26 @@ package main import ( "context" - "database/sql" "errors" "fmt" "strings" "github.com/franchoy/coldkeep/internal/batch" - "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/recovery" "github.com/franchoy/coldkeep/internal/storage" ) -var connectListSearchDBPhase = db.ConnectDB - -var newCommandEngine = func(dbconn *sql.DB, containerDir string) (engine.Engine, error) { - return engine.New(engine.Config{DB: dbconn, ContainerDir: containerDir}) -} - -var newSnapshotRestoreCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { - return engine.New(engine.Config{ - DB: sgctx.DB, - ContainerDir: sgctx.EffectiveContainerDir(), - StoreContext: &sgctx, - }) -} - -var newStoreFolderCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { - return engine.New(engine.Config{ - DB: sgctx.DB, - ContainerDir: sgctx.EffectiveContainerDir(), - StoreContext: &sgctx, - }) -} - -var newConfigurationCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { - return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) -} - -var newSnapshotReadCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { - return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) -} - -var newVerifyCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { - return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) -} - -var newObservabilityCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { - return engine.New(engine.Config{DB: sgctx.DB, ContainerDir: sgctx.EffectiveContainerDir()}) -} - -var newRepairCommandEngine = func(dbconn *sql.DB) (engine.Engine, error) { - return engine.New(engine.Config{DB: dbconn}) -} - -var connectRepairDBPhase = db.ConnectDB - -var connectRecoveryDBPhase = db.ConnectDB - -var newRecoveryCommandEngine = func(dbconn *sql.DB, containersDir string) (engine.Engine, error) { - return engine.New(engine.Config{DB: dbconn, ContainerDir: containersDir}) -} - func runRecoveryThroughEngine(containersDir string) (recovery.Report, error) { - dbconn, err := connectRecoveryDBPhase() + session, err := openCommandSession("recovery", false, containersDir) if err != nil { return recovery.Report{}, fmt.Errorf("failed to connect to DB: %w", err) } - defer func() { _ = dbconn.Close() }() - eng, err := newRecoveryCommandEngine(dbconn, containersDir) - if err != nil { - return recovery.Report{}, err - } - ctx, cancel := db.NewOperationContext(context.Background()) + defer func() { _ = session.Close() }() + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - result, err := eng.Recover(ctx, engine.RecoverRequest{}) + result, err := session.Engine().Recover(ctx, engine.RecoverRequest{}) return recoveryReportFromEngine(result), err } @@ -91,25 +35,15 @@ func recoveryReportFromEngine(result engine.RecoverResult) recovery.Report { } } -var connectDoctorDBPhase = db.ConnectDB - -var newDoctorCommandEngine = func(dbconn *sql.DB, containersDir string) (engine.Engine, error) { - return engine.New(engine.Config{DB: dbconn, ContainerDir: containersDir}) -} - func executeDoctorEngine(containersDir, verifyLevel string) (engine.DoctorResult, error) { - dbconn, err := connectDoctorDBPhase() + session, err := openCommandSession("doctor", false, containersDir) if err != nil { return engine.DoctorResult{}, fmt.Errorf("failed to connect to DB: %w", err) } - defer func() { _ = dbconn.Close() }() - eng, err := newDoctorCommandEngine(dbconn, containersDir) - if err != nil { - return engine.DoctorResult{}, err - } - ctx, cancel := db.NewOperationContext(context.Background()) + defer func() { _ = session.Close() }() + ctx, cancel := session.OperationContext(context.Background()) defer cancel() - return eng.Doctor(ctx, engine.DoctorRequest{VerifyLevel: verifyLevel}) + return session.Engine().Doctor(ctx, engine.DoctorRequest{VerifyLevel: verifyLevel}) } func restoreStoredPathWithEngine( diff --git a/cmd/coldkeep/thin_cli_dependency_guard_test.go b/cmd/coldkeep/thin_cli_dependency_guard_test.go new file mode 100644 index 00000000..a5e6eb46 --- /dev/null +++ b/cmd/coldkeep/thin_cli_dependency_guard_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" +) + +// TestProductionCLIUsesOnlyApplicationAndEngineExecutionBoundaries is the +// executable thin-CLI contract. Compatibility projection and explicitly +// isolated benchmark/simulation tooling have exact, reviewable exceptions. +func TestProductionCLIUsesOnlyApplicationAndEngineExecutionBoundaries(t *testing.T) { + _, current, _, _ := runtime.Caller(0) + dir := filepath.Dir(current) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + allowed := map[string]bool{ + "application_session.go:openApplicationSession:application.Open": true, + "init.go:initCommand:storage.IsRegisteredCompressionCodec": true, + "main.go:resolveTraceOptions:observability.NewJSONTraceSink": true, + "main.go:newSimulationObservabilityService:observability.NewService": true, + "main.go:captureBenchmarkDiagnosticFinalState:db.BuildPostgresConnStringFromEnv": true, + "main.go:createTemporaryBenchmarkDatabase:db.BuildPostgresConnStringFromEnv": true, + "main.go:captureBenchmarkState:db.BuildPostgresConnStringFromEnv": true, + "main.go:runSimulateCommand:storage.ParseStorageContext": true, + "main.go:runSimulateCommand:storage.StoreFileWithStorageContext": true, + "main.go:runSimulateCommand:storage.StoreFileWithStorageContextAndCodec": true, + "main.go:runSimulateCommand:storage.StoreFolderWithStorageContextAndOptions": true, + "main.go:runSimulateCommand:storage.StoreFolderWithStorageContextAndCodecAndOptions": true, + "main.go:parseSnapshotQuery:snapshot.NormalizeSnapshotPath": true, + "main.go:parseSnapshotRestoreExactPathSelectors:snapshot.NormalizeSnapshotPath": true, + "main.go:parseSnapshotRestorePrefixSelectors:snapshot.NormalizeSnapshotPath": true, + } + used := make(map[string]bool) + var violations []string + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(dir, name), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + inspectCalls := func(owner string, body ast.Node) { + ast.Inspect(body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := selector.X.(*ast.Ident) + if !ok || !isGuardedCLIPackage(pkg.Name) { + return true + } + key := name + ":" + owner + ":" + pkg.Name + "." + selector.Sel.Name + if allowed[key] { + used[key] = true + return true + } + if isProjectionOnlyCall(name, pkg.Name, selector.Sel.Name) { + return true + } + violations = append(violations, key) + return true + }) + } + for _, declaration := range file.Decls { + switch declaration := declaration.(type) { + case *ast.FuncDecl: + if declaration.Body != nil { + inspectCalls(declaration.Name.Name, declaration.Body) + } + case *ast.GenDecl: + for _, spec := range declaration.Specs { + values, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for i, value := range values.Values { + literal, ok := value.(*ast.FuncLit) + if !ok { + continue + } + owner := "" + if i < len(values.Names) { + owner = values.Names[i].Name + } + inspectCalls(owner, literal.Body) + } + } + } + } + } + for exception := range allowed { + if !used[exception] { + violations = append(violations, "stale allowlist entry: "+exception) + } + } + sort.Strings(violations) + if len(violations) > 0 { + t.Fatalf("production CLI bypasses application/engine boundary:\n %s", strings.Join(violations, "\n ")) + } +} + +func isGuardedCLIPackage(name string) bool { + switch name { + case "application", "catalog", "db", "maintenance", "observability", "recovery", "snapshot", "storage": + return true + default: + return false + } +} + +func isProjectionOnlyCall(filename, packageName, selector string) bool { + if filename == "observability_engine_adapters.go" && packageName == "observability" { + return true + } + if filename == "stored_path_engine_adapters.go" && packageName == "storage" { + return true + } + if filename != "main.go" || packageName != "snapshot" { + return false + } + switch selector { + case "Snapshot", "SnapshotLineageStatus", "DiffType": + return true + default: + return false + } +} diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 6986549c..3333d4cf 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–17 are complete; Phase 18 is next. + 0–18 are complete; Phase 19 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 117b63f2..ac562996 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -117,11 +117,13 @@ ## Phase 18 — Application Composition and Thin CLI Closure -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 18 application composition and thin CLI closure](v1.13.12-phase18-application-composition-and-thin-cli-closure.md). ## Phase 19 — Dependency and Contract Hardening -**Status:** Not started +**Status:** Next ## Phase 20 — Compatibility and Adversarial Regression diff --git a/docs/release/v1.13/v1.13.12-phase18-application-composition-and-thin-cli-closure.md b/docs/release/v1.13/v1.13.12-phase18-application-composition-and-thin-cli-closure.md new file mode 100644 index 00000000..3247135c --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase18-application-composition-and-thin-cli-closure.md @@ -0,0 +1,86 @@ +# Coldkeep v1.13.12 Phase 18 — Application Composition and Thin CLI Closure + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `18 — Application Composition and Thin CLI Closure` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Production ownership + +`internal/application` is now the single production composition boundary for +configured database/storage opening, engine construction, operation-context +timeouts, and resource closure. It preserves the configured backend and +storage resolution, does not acquire the repository coordination lease, and +does not select a new default backend. The CLI still validates arguments and +acquires the outer lease before opening a session. + +All normal repository commands now open an application session and call its +injected engine. Store and folder-store receive the writer-bearing storage +context only inside composition. Configuration, current-file queries, +snapshot reads and mutations, stats, inspect, verify, GC, repair, recovery, +startup recovery, and Doctor no longer open DB/storage contexts or construct +engines in production command code. By-ID restore and remove now issue one +batch engine request per command rather than orchestrating per-item engine +calls in the CLI; `RestoreItemResult.OriginalName` preserves the established +batch projection without a second metadata query. + +Live-repository `simulate gc` now calls the typed read-only +`Engine.PlanGarbageCollection` operation. Its neutral result preserves exact +reachability, reclaimability, packed-block, container-impact, warning, and +hypothetical snapshot-omission data for the existing observability renderer. +The isolated store/store-folder mutation simulator and benchmark/determinism +harnesses remain explicitly classified tooling exclusions. + +## Enforcement + +An AST guard walks every production CLI Go file and rejects direct calls into +DB, catalog, maintenance, recovery, observability services, snapshot +execution, storage execution, or application construction outside the exact +composition point. It also rejects stale allowlist entries. The narrow +allowlist covers presentation trace sinks, snapshot path parsing, init-time +codec validation, exact benchmark database helpers, and isolated simulated +storage. Test-only legacy construction seams preserve frozen compatibility +fixtures but are absent from production builds. + +The dependency-direction guard now recognizes the intended application → +engine composition edge while retaining the prohibition for other internal +domain packages. + +## Compatibility + +CLI syntax, text and JSON projections, exit codes, startup/coordination order, +operation timeouts, transaction and locking behavior, storage bytes, +repository format, schema and migrations, backend selection, and SQLite/ +PostgreSQL support remain unchanged. No SQLite-default transition, storage +rewrite, or broader coordination work was performed. + +## Validation + +- Production build and AST thin-CLI guard: pass. +- Direct application-session and GC-plan engine contracts: pass. +- Frozen CLI routing/projection fixtures, including legacy test seams: pass. +- Full isolated `go test -count=1 ./...`: pass. +- Focused CLI/application/engine race suites: pass. +- Shared SQLite/PostgreSQL GC-plan contract: pass. +- Validation root: `/tmp/coldkeep-v11312-phase18.i3cOLh/`. + +One full-suite attempt reported a failure confined to `internal/storage` after +all focused Phase 18 suites had passed. An isolated complete rerun of that +package passed, and the subsequent complete `go test -count=1 ./...` stage +also passed at the identical tracked candidate state. No source, fixture, +storage, or database state changed between attempts; the retained logs record +the nonzero attempt and complete-stage environmental retry. + +PostgreSQL validation created and dropped exactly: + +- `coldkeep_bt_testenginegarbagecollectionplanacrossbacke_35a722_1` + +A read-only maintenance query confirmed the exact name is absent. No ordinary +database was reset, migrated, truncated, or dropped. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 19 is authorized to harden dependency direction, complete neutral +contract/reflection coverage, run the comprehensive dual-backend contract +matrix, and remove remaining promised-but-unimplemented or stale surfaces. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index c1247683..51e3f1ec 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,9 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–17 are Complete. Phase 18 is Next: introduce the application -composition/session boundary and close the production CLI to parsing, -projection, rendering, coordination, and application opening only. +Phases 0–18 are Complete. Phase 19 is Next: harden dependency direction and +neutral contracts, run the comprehensive engine/catalog dual-backend matrix, +and remove remaining promised-but-unimplemented or stale surfaces. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 3384e98b..676d69a3 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -274,11 +274,26 @@ ## Phase 18 — Application Composition and Thin CLI Closure -**Phase status:** Next +**Phase status:** Complete + +- [x] One application session owns configured DB/storage opening, injected + engine construction, operation timeout contexts, and resource closure while + leaving backend choice and outer coordination unchanged. +- [x] Production repository commands open application sessions and no longer + connect DB/storage or construct engines directly. +- [x] By-ID restore/remove use one engine batch request per command and retain + frozen batch reports without a CLI metadata query. +- [x] Live-repository GC simulation crosses a typed, read-only engine planning + boundary; isolated mutation simulation and benchmark tooling remain exact + exclusions. +- [x] An AST guard rejects direct lower-layer production calls and stale or + unreviewed allowlist entries. +- [x] Full isolated regression, focused race, and shared SQLite/PostgreSQL + GC-plan contract suites pass. ## Phase 19 — Dependency and Contract Hardening -**Phase status:** Not started +**Phase status:** Next ## Phase 20 — Compatibility and Adversarial Regression diff --git a/internal/application/session.go b/internal/application/session.go new file mode 100644 index 00000000..d30cc345 --- /dev/null +++ b/internal/application/session.go @@ -0,0 +1,98 @@ +// Package application owns runtime composition of configured database, +// storage, and engine dependencies. It deliberately does not acquire the +// repository coordination lease or choose a different backend default. +package application + +import ( + "context" + "database/sql" + "errors" + "strings" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/storage" +) + +// Request describes the dependencies needed by one application operation. +type Request struct { + Operation string + ContainerDir string + RequireStorage bool +} + +// Session owns exactly one configured runtime connection/context and its +// injected engine. Close must be called by the opener. +type Session struct { + engine engine.Engine + db *sql.DB + storage *storage.StorageContext +} + +// Open constructs one engine using the existing configured backend and +// storage resolution. Backend selection and startup coordination remain with +// their established owners. +func Open(req Request) (*Session, error) { + if req.RequireStorage { + storageContext, err := storage.LoadDefaultStorageContext() + if err != nil { + return nil, err + } + containerDir := storageContext.EffectiveContainerDir() + if strings.TrimSpace(req.ContainerDir) != "" { + containerDir = req.ContainerDir + } + eng, err := engine.New(engine.Config{ + DB: storageContext.DB, ContainerDir: containerDir, + StoreContext: &storageContext, + }) + if err != nil { + _ = storageContext.Close() + return nil, err + } + return &Session{engine: eng, storage: &storageContext}, nil + } + + dbconn, err := db.ConnectDB() + if err != nil { + return nil, err + } + eng, err := engine.New(engine.Config{DB: dbconn, ContainerDir: req.ContainerDir}) + if err != nil { + _ = dbconn.Close() + return nil, err + } + return &Session{engine: eng, db: dbconn}, nil +} + +// Engine returns the injected headless engine. +func (s *Session) Engine() engine.Engine { + if s == nil { + return nil + } + return s.engine +} + +// OperationContext applies the established repository-operation timeout +// without exposing database helpers to command code. +func (s *Session) OperationContext(parent context.Context) (context.Context, context.CancelFunc) { + return db.NewOperationContext(parent) +} + +// Close releases the session-owned resources. +func (s *Session) Close() error { + if s == nil { + return nil + } + var closeErr error + if s.storage != nil { + closeErr = s.storage.Close() + s.storage = nil + } + if s.db != nil { + closeErr = errors.Join(closeErr, s.db.Close()) + s.db = nil + } + s.engine = nil + return closeErr +} diff --git a/internal/application/session_test.go b/internal/application/session_test.go new file mode 100644 index 00000000..ab772ae0 --- /dev/null +++ b/internal/application/session_test.go @@ -0,0 +1,21 @@ +package application + +import ( + "context" + "testing" +) + +func TestNilSessionIsSafe(t *testing.T) { + var session *Session + if session.Engine() != nil { + t.Fatal("nil session returned an engine") + } + if err := session.Close(); err != nil { + t.Fatalf("nil session close: %v", err) + } + ctx, cancel := session.OperationContext(context.Background()) + defer cancel() + if ctx == nil { + t.Fatal("nil session operation context is nil") + } +} diff --git a/internal/engine/candidates.go b/internal/engine/candidates.go index db8b5820..7feb4957 100644 --- a/internal/engine/candidates.go +++ b/internal/engine/candidates.go @@ -251,6 +251,10 @@ type RestoreRequest struct { type RestoreItemResult struct { // FileID is the restored logical file ID. FileID int64 + // OriginalName is the persisted logical-file name used to derive the + // destination. It lets adapters preserve the established batch projection + // without querying storage metadata outside the engine boundary. + OriginalName string // DestinationPath is the path the file was (or would be) written to. DestinationPath string // RestoredHash is the content hash of the restored file. @@ -447,6 +451,50 @@ type GarbageCollectResult struct { Warnings []OperationWarning } +// GarbageCollectionPlanRequest describes a read-only live-repository GC plan. +type GarbageCollectionPlanRequest struct { + SnapshotIDsToOmit []string + IncludeTrace bool +} + +// GarbageCollectionContainerImpact is a presentation-neutral container plan. +type GarbageCollectionContainerImpact struct { + ContainerID int64 + Filename string + TotalBytes int64 + LiveBytesAfterGC int64 + ReclaimableBytes int64 + ReclaimableChunks int64 + TotalChunks int64 + FullyReclaimable bool + RequiresCompaction bool +} + +// GarbageCollectionPlanSummary preserves exact GC planner counters. +type GarbageCollectionPlanSummary struct { + TotalChunks int64 + ReachableChunks int64 + UnreachableChunks int64 + LogicallyReclaimableBytes int64 + PhysicallyReclaimableBytes int64 + FullyReclaimableContainers int64 + PartiallyDeadContainers int64 + PackedBlocksLive int64 + PackedBlocksDead int64 + PackedBytesLive int64 + PackedBytesReclaimable int64 + RetainedDeadBytesDueToPackedBlocks int64 +} + +// GarbageCollectionPlanResult is the immutable read-only plan projection. +type GarbageCollectionPlanResult struct { + SnapshotIDsToOmit []string + Summary GarbageCollectionPlanSummary + Containers []GarbageCollectionContainerImpact + Warnings []OperationWarning + Trace []TraceEvent +} + // --------------------------------------------------------------------------- // Snapshot operations // --------------------------------------------------------------------------- diff --git a/internal/engine/candidates_test.go b/internal/engine/candidates_test.go index 53bcc2b6..3e6e2b82 100644 --- a/internal/engine/candidates_test.go +++ b/internal/engine/candidates_test.go @@ -24,7 +24,7 @@ func TestEngineActiveInterfaceApprovedMethods(t *testing.T) { got[typ.Method(i).Name] = true } - want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Recover", "Doctor", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} + want := []string{"Stats", "Inspect", "Verify", "SnapshotList", "SnapshotShow", "SnapshotStats", "SnapshotDiff", "SnapshotCreate", "SnapshotDelete", "SnapshotRestore", "GarbageCollect", "PlanGarbageCollection", "Store", "StoreFolder", "ListFiles", "SearchFiles", "GetConfiguration", "SetConfiguration", "Repair", "Recover", "Doctor", "Remove", "RemoveStoredPaths", "Restore", "RestoreStoredPath"} for _, name := range want { if !got[name] { t.Errorf("Engine interface missing expected method %q", name) diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 2b57aac6..2fc76130 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -66,6 +66,10 @@ func activeCoreContractTypes() []struct { {"RemoveStoredPathsResult", engine.RemoveStoredPathsResult{}}, {"GarbageCollectRequest", engine.GarbageCollectRequest{}}, {"GarbageCollectResult", engine.GarbageCollectResult{}}, + {"GarbageCollectionPlanRequest", engine.GarbageCollectionPlanRequest{}}, + {"GarbageCollectionPlanSummary", engine.GarbageCollectionPlanSummary{}}, + {"GarbageCollectionContainerImpact", engine.GarbageCollectionContainerImpact{}}, + {"GarbageCollectionPlanResult", engine.GarbageCollectionPlanResult{}}, {"VerifyRequest", engine.VerifyRequest{}}, {"VerifyResult", engine.VerifyResult{}}, {"RepairRequest", engine.RepairRequest{}}, diff --git a/internal/engine/default_engine_routing_helpers.go b/internal/engine/default_engine_routing_helpers.go index fa656edf..694fdde6 100644 --- a/internal/engine/default_engine_routing_helpers.go +++ b/internal/engine/default_engine_routing_helpers.go @@ -9,8 +9,10 @@ import ( "strings" "github.com/franchoy/coldkeep/internal/blocks" + "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/container" "github.com/franchoy/coldkeep/internal/execution" + internalgc "github.com/franchoy/coldkeep/internal/gc" "github.com/franchoy/coldkeep/internal/invariants" "github.com/franchoy/coldkeep/internal/maintenance" "github.com/franchoy/coldkeep/internal/snapshot" @@ -131,6 +133,93 @@ func (e *DefaultEngine) GarbageCollect(ctx context.Context, req GarbageCollectRe }, nil } +func (e *DefaultEngine) PlanGarbageCollection(ctx context.Context, req GarbageCollectionPlanRequest) (GarbageCollectionPlanResult, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return GarbageCollectionPlanResult{}, err + } + result := GarbageCollectionPlanResult{SnapshotIDsToOmit: append([]string(nil), req.SnapshotIDsToOmit...)} + if req.IncludeTrace { + result.Trace = append(result.Trace, TraceEvent{ + Step: "simulate.gc.start", Message: "starting gc simulation", + Metadata: map[string]Value{"assumed_deleted_snapshots": integerValue(len(req.SnapshotIDsToOmit))}, + }) + rootMetadata := map[string]Value{"excluded_snapshots": integerValue(len(req.SnapshotIDsToOmit))} + if roots, rootsErr := catalog.NewServiceFromSQL(e.config.DB).LoadGCPlanMetadata(ctx, catalog.GCPlanInput{ExcludeSnapshotIDs: req.SnapshotIDsToOmit}); rootsErr == nil { + rootMetadata["root_count"] = integerValue(len(roots.Roots)) + } + result.Trace = append(result.Trace, TraceEvent{Step: "simulate.gc.roots.load", Message: "loading gc roots", Metadata: rootMetadata}) + for _, snapshotID := range req.SnapshotIDsToOmit { + result.Trace = append(result.Trace, TraceEvent{ + Step: "simulate.gc.assumption.exclude_snapshot", Entity: "snapshot", EntityID: snapshotID, + Message: "excluding snapshot from simulation roots", + }) + } + result.Trace = append(result.Trace, TraceEvent{Step: "simulate.gc.mark.start", Message: "starting reachable chunk mark traversal"}) + } + plan, err := internalgc.BuildPlan(ctx, e.config.DB, internalgc.PlanOptions{ + AssumeDeletedSnapshots: append([]string(nil), req.SnapshotIDsToOmit...), + }) + if err != nil { + return result, err + } + result.Summary = GarbageCollectionPlanSummary{ + TotalChunks: plan.TotalChunks, ReachableChunks: plan.ReachableChunks, + UnreachableChunks: plan.Summary.UnreachableChunks, + LogicallyReclaimableBytes: plan.Summary.LogicallyReclaimableBytes, + PhysicallyReclaimableBytes: plan.Summary.PhysicallyReclaimableBytes, + FullyReclaimableContainers: plan.Summary.FullyReclaimableContainers, + PartiallyDeadContainers: plan.Summary.PartiallyDeadContainers, + PackedBlocksLive: plan.Summary.PackedBlocksLive, PackedBlocksDead: plan.Summary.PackedBlocksDead, + PackedBytesLive: plan.Summary.PackedBytesLive, PackedBytesReclaimable: plan.Summary.PackedBytesReclaimable, + RetainedDeadBytesDueToPackedBlocks: plan.Summary.RetainedDeadBytesDueToPackedBlocks, + } + if req.IncludeTrace { + result.Trace = append(result.Trace, + TraceEvent{Step: "simulate.gc.mark.complete", Message: "reachable chunk set computed", Metadata: map[string]Value{"reachable_chunks": integerValue(plan.ReachableChunks)}}, + TraceEvent{Step: "simulate.gc.unreachable.compute", Message: "computed unreachable chunk set and logical reclaimability", Metadata: map[string]Value{ + "unreachable_chunks": integerValue(plan.Summary.UnreachableChunks), "logically_reclaimable_bytes": integerValue(plan.Summary.LogicallyReclaimableBytes), + }}, + ) + } + for _, item := range plan.AffectedContainers { + result.Containers = append(result.Containers, GarbageCollectionContainerImpact{ + ContainerID: item.ContainerID, Filename: item.Filename, TotalBytes: item.TotalBytes, + LiveBytesAfterGC: item.LiveBytesAfterGC, ReclaimableBytes: item.ReclaimableBytes, + ReclaimableChunks: item.ReclaimableChunks, TotalChunks: item.TotalChunks, + FullyReclaimable: item.FullyReclaimable, RequiresCompaction: item.RequiresCompaction, + }) + } + for _, warning := range plan.Warnings { + result.Warnings = append(result.Warnings, OperationWarning{Code: warning.Code, Message: warning.Message}) + } + if req.IncludeTrace { + result.Trace = append(result.Trace, + TraceEvent{Step: "simulate.gc.container_impact.compute", Message: "computed per-container reclaim impact", Metadata: map[string]Value{ + "affected_containers": integerValue(len(plan.AffectedContainers)), + "fully_reclaimable_containers": integerValue(plan.Summary.FullyReclaimableContainers), + "partially_dead_containers": integerValue(plan.Summary.PartiallyDeadContainers), + }}, + TraceEvent{Step: "simulate.gc.complete", Message: "completed gc simulation", Metadata: map[string]Value{ + "reachable_chunks": integerValue(result.Summary.ReachableChunks), "unreachable_chunks": integerValue(result.Summary.UnreachableChunks), + "affected_containers": integerValue(len(result.Containers)), "warnings": integerValue(len(result.Warnings)), + "physically_reclaimable_bytes": integerValue(result.Summary.PhysicallyReclaimableBytes), + }}, + ) + } + return result, nil +} + +func integerValue(value any) Value { + converted, err := valueFromAny(value) + if err != nil { + panic(err) + } + return converted +} + func (e *DefaultEngine) Store(ctx context.Context, req StoreRequest) (StoreResult, error) { if err := ctx.Err(); err != nil { return StoreResult{}, err @@ -323,7 +412,7 @@ func (e *DefaultEngine) dryRunRestoreFileID(req RestoreRequest, fileID int64) Re func (e *DefaultEngine) liveRestoreFileID(req RestoreRequest, fileID int64) RestoreItemResult { sgctx := storage.StorageContext{DB: e.config.DB, ContainerDir: e.config.ContainerDir} - out, err := restoreByIDOutputPath(e.config.DB, fileID, req.DestinationRoot) + out, originalName, err := restoreByIDOutputPath(e.config.DB, fileID, req.DestinationRoot) if err != nil { return failedRestoreItem(fileID, err.Error()) } @@ -336,6 +425,7 @@ func (e *DefaultEngine) liveRestoreFileID(req RestoreRequest, fileID int64) Rest } return RestoreItemResult{ FileID: fileID, + OriginalName: originalName, Status: BatchItemOK, DestinationPath: r.OutputPath, RestoredHash: r.RestoredHash, @@ -364,11 +454,12 @@ func finalizeBatchSummary(summary *BatchSummary, requested int) { func dryRunRestoreByID(dbconn *sql.DB, fileID int64, outputDir string, overwrite bool) (RestoreItemResult, error) { item := RestoreItemResult{FileID: fileID, Status: BatchItemOK} - out, err := restoreByIDOutputPath(dbconn, fileID, outputDir) + out, originalName, err := restoreByIDOutputPath(dbconn, fileID, outputDir) if err != nil { return item, err } item.DestinationPath = out + item.OriginalName = originalName if !overwrite { if _, statErr := os.Stat(out); statErr == nil { return item, fmt.Errorf("output file already exists: %s (use --overwrite)", out) @@ -379,15 +470,15 @@ func dryRunRestoreByID(dbconn *sql.DB, fileID int64, outputDir string, overwrite return item, nil } -func restoreByIDOutputPath(dbconn *sql.DB, fileID int64, outputDir string) (string, error) { +func restoreByIDOutputPath(dbconn *sql.DB, fileID int64, outputDir string) (string, string, error) { info, err := storage.GetLogicalFileInfoWithDB(dbconn, fileID) if err != nil { - return "", err + return "", "", err } if info.Status != filestate.LogicalFileCompleted { - return "", fmt.Errorf("file ID %d is not COMPLETED", fileID) + return "", "", fmt.Errorf("file ID %d is not COMPLETED", fileID) } - return filepath.Join(outputDir, info.OriginalName), nil + return filepath.Join(outputDir, info.OriginalName), info.OriginalName, nil } func dryRunRemoveByID(dbconn *sql.DB, fileID int64) error { diff --git a/internal/engine/dependency_guard_test.go b/internal/engine/dependency_guard_test.go index 05d2c2c2..a3a395c0 100644 --- a/internal/engine/dependency_guard_test.go +++ b/internal/engine/dependency_guard_test.go @@ -17,7 +17,8 @@ type goListPackage struct { // TestEngineDependencyDirection enforces the v1.11 engine facade boundary: // // - Rule 1: internal/engine must not import cmd/coldkeep (transitively). -// - Rule 2: non-engine internal/* packages must not directly import internal/engine. +// - Rule 2: non-engine internal/* packages except the application composition +// boundary must not directly import internal/engine. // - Rule 3: cmd/coldkeep may import internal/engine (allowed; not tested here). // // This guard prevents architecture drift as more CLI commands are routed through @@ -27,6 +28,7 @@ func TestEngineDependencyDirection(t *testing.T) { const enginePkg = module + "/internal/engine" const catalogPkg = module + "/internal/catalog" const cliPkg = module + "/cmd/coldkeep" + const applicationPkg = module + "/internal/application" // Locate the module root so ./... covers all packages, not just internal/engine. modOut, err := exec.Command("go", "env", "GOMOD").Output() @@ -49,7 +51,7 @@ func TestEngineDependencyDirection(t *testing.T) { t.Fatalf("decode go list output: %v", err) } checkEngineNotDependsOnCLI(t, pkg, enginePkg, cliPkg) - checkDomainNotImportsEngine(t, pkg, module, enginePkg) + checkDomainNotImportsEngine(t, pkg, module, enginePkg, applicationPkg) checkCLINotImportsCatalog(t, pkg, cliPkg, catalogPkg) } } @@ -71,10 +73,10 @@ func checkEngineNotDependsOnCLI(t *testing.T, pkg goListPackage, enginePkg, cliP // checkDomainNotImportsEngine enforces Rule 2: non-engine internal/* packages // must not directly import internal/engine. -func checkDomainNotImportsEngine(t *testing.T, pkg goListPackage, module, enginePkg string) { +func checkDomainNotImportsEngine(t *testing.T, pkg goListPackage, module, enginePkg, applicationPkg string) { t.Helper() if !strings.HasPrefix(pkg.ImportPath, module+"/internal/") || - strings.HasPrefix(pkg.ImportPath, enginePkg) { + strings.HasPrefix(pkg.ImportPath, enginePkg) || pkg.ImportPath == applicationPkg { return } for _, imp := range pkg.Imports { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 0ae84041..4a90d595 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -74,6 +74,10 @@ type Engine interface { // on both backends. GarbageCollect(ctx context.Context, req GarbageCollectRequest) (GarbageCollectResult, error) + // PlanGarbageCollection computes live-repository reachability and reclaim + // impact without mutating database or storage state. + PlanGarbageCollection(ctx context.Context, req GarbageCollectionPlanRequest) (GarbageCollectionPlanResult, error) + // Store stores a file into the repository. // Safety invariant: Store must not create inconsistent catalog/storage state. // Active semantics are limited to single-file store. diff --git a/internal/engine/gc_engine_test.go b/internal/engine/gc_engine_test.go index 4ba3a9bb..cf93091d 100644 --- a/internal/engine/gc_engine_test.go +++ b/internal/engine/gc_engine_test.go @@ -2,6 +2,7 @@ package engine_test import ( "context" + "errors" "strings" "testing" @@ -35,6 +36,31 @@ func TestGCDryRunThroughEngineEmptyDB(t *testing.T) { } } +func TestEnginePlanGarbageCollectionEmptyAndCancellation(t *testing.T) { + db := openSnapshotTestDB(t) + eng, err := engine.New(engine.Config{DB: db, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + + result, err := eng.PlanGarbageCollection(context.Background(), engine.GarbageCollectionPlanRequest{IncludeTrace: true}) + if err != nil { + t.Fatalf("PlanGarbageCollection: %v", err) + } + if result.Summary.TotalChunks != 0 || result.Summary.ReachableChunks != 0 || result.Summary.UnreachableChunks != 0 { + t.Fatalf("unexpected empty plan: %+v", result.Summary) + } + if len(result.Trace) != 7 || result.Trace[0].Step != "simulate.gc.start" || result.Trace[len(result.Trace)-1].Step != "simulate.gc.complete" { + t.Fatalf("unexpected plan trace: %+v", result.Trace) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := eng.PlanGarbageCollection(cancelled, engine.GarbageCollectionPlanRequest{}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled PlanGarbageCollection: %v", err) + } +} + // TestGCDryRunEchoesFields verifies that GarbageCollectResult fields are // correctly populated from the underlying maintenance.GCResult (field mapping). func TestGCDryRunEchoesFields(t *testing.T) { diff --git a/internal/engine/gc_plan_backend_contract_test.go b/internal/engine/gc_plan_backend_contract_test.go new file mode 100644 index 00000000..30d30809 --- /dev/null +++ b/internal/engine/gc_plan_backend_contract_test.go @@ -0,0 +1,25 @@ +package engine_test + +import ( + "context" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineGarbageCollectionPlanAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + eng, err := engine.New(engine.Config{DB: backend.DB, ContainerDir: t.TempDir()}) + if err != nil { + t.Fatalf("new engine: %v", err) + } + result, err := eng.PlanGarbageCollection(context.Background(), engine.GarbageCollectionPlanRequest{}) + if err != nil { + t.Fatalf("PlanGarbageCollection: %v", err) + } + if result.Summary != (engine.GarbageCollectionPlanSummary{}) || len(result.Containers) != 0 || len(result.Warnings) != 0 { + t.Fatalf("empty repository plan=%+v", result) + } + }) +} From 27505fdd3c6200ed567bbf389daca5ce09a015d1 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 16:22:44 +0200 Subject: [PATCH 20/27] Harden engine and catalog contracts --- .../read_side_taxonomy_boundary_test.go | 12 +- .../thin_cli_dependency_guard_test.go | 91 +++++++++-- docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...ase19-dependency-and-contract-hardening.md | 94 +++++++++++ ...-engine-error-and-neutrality-foundation.md | 7 +- ....12-phase4-catalog-contract-replacement.md | 7 +- ...se9-gc-plan-implementation-and-adoption.md | 4 +- docs/release/v1.13/v1.13.12-scope.md | 7 +- .../v1.13/v1.13.12-validation-checklist.md | 21 ++- internal/catalog/catalog.go | 9 +- internal/catalog/contracts_test.go | 3 - internal/catalog/errors.go | 41 ++--- internal/catalog/errors_test.go | 46 ++---- internal/catalog/neutrality_test.go | 84 ++++++++-- internal/engine/contracts_test.go | 146 ++++++++++++++++++ ...ent_configuration_backend_contract_test.go | 87 +++++++++++ internal/engine/current_files.go | 6 +- internal/engine/current_files_test.go | 2 +- internal/engine/default_engine.go | 59 +++++-- .../engine/default_engine_routing_helpers.go | 25 ++- internal/engine/dependency_guard_test.go | 22 +++ internal/engine/doctor.go | 3 +- internal/engine/engine.go | 7 +- internal/engine/errors.go | 17 +- internal/engine/errors_test.go | 21 +-- .../engine/read_side_backend_contract_test.go | 11 +- .../read_side_taxonomy_boundary_test.go | 48 +++--- internal/engine/recover.go | 3 +- internal/engine/repair.go | 3 +- internal/engine/repository_configuration.go | 6 +- internal/engine/restore_stored_path.go | 5 +- internal/engine/snapshot_create.go | 7 +- internal/engine/snapshot_delete.go | 5 +- internal/engine/snapshot_restore.go | 5 +- .../engine/validation_vs_unsupported_test.go | 21 --- .../read_side_taxonomy_boundary_test.go | 16 +- 37 files changed, 696 insertions(+), 263 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase19-dependency-and-contract-hardening.md create mode 100644 internal/engine/current_configuration_backend_contract_test.go diff --git a/cmd/coldkeep/read_side_taxonomy_boundary_test.go b/cmd/coldkeep/read_side_taxonomy_boundary_test.go index 38ce1185..9b943dc0 100644 --- a/cmd/coldkeep/read_side_taxonomy_boundary_test.go +++ b/cmd/coldkeep/read_side_taxonomy_boundary_test.go @@ -7,13 +7,12 @@ import ( "strings" "testing" - "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/snapshot" "github.com/franchoy/coldkeep/internal/storage" ) -func TestReadSideCLIValidationErrorsRemainOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestReadSideCLIValidationErrorsRemainOutsideUnsupportedClassification(t *testing.T) { t.Run("inspect unsupported entity", func(t *testing.T) { err := runInspectCommand(parsedCommandLine{ method: "inspect", @@ -48,7 +47,7 @@ func TestReadSideCLIValidationErrorsRemainOutsideUnsupportedAndDeferredClassific }) } -func TestSnapshotDiffInvalidRegexRemainsOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestSnapshotDiffInvalidRegexRemainsOutsideUnsupportedClassification(t *testing.T) { err := runSnapshotCommand(parsedCommandLine{ method: "snapshot", positionals: []string{"diff", "snap-1", "snap-2"}, @@ -57,7 +56,7 @@ func TestSnapshotDiffInvalidRegexRemainsOutsideUnsupportedAndDeferredClassificat assertCLIReadSideBoundaryError(t, err, "invalid --regex value") } -func TestSnapshotShowMissingSnapshotRemainsOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestSnapshotShowMissingSnapshotRemainsOutsideUnsupportedClassification(t *testing.T) { originalLoad := loadDefaultStorageContextPhase originalGet := getSnapshotPhase t.Cleanup(func() { @@ -83,7 +82,7 @@ func TestSnapshotShowMissingSnapshotRemainsOutsideUnsupportedAndDeferredClassifi assertCLIReadSideBoundaryError(t, err, `snapshot "missing-snap" not found`) } -func TestSnapshotDiffMissingSnapshotRemainsOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestSnapshotDiffMissingSnapshotRemainsOutsideUnsupportedClassification(t *testing.T) { originalLoad := loadDefaultStorageContextPhase originalDiff := diffSnapshotsPhase t.Cleanup(func() { @@ -121,7 +120,4 @@ func assertCLIReadSideBoundaryError(t *testing.T, err error, wantSubstring strin if engine.IsUnsupported(err) { t.Fatalf("expected CLI read-side error to remain outside unsupported classification: %v", err) } - if catalog.IsDeferred(err) { - t.Fatalf("expected CLI read-side error to remain outside deferred classification: %v", err) - } } diff --git a/cmd/coldkeep/thin_cli_dependency_guard_test.go b/cmd/coldkeep/thin_cli_dependency_guard_test.go index a5e6eb46..b23d05d1 100644 --- a/cmd/coldkeep/thin_cli_dependency_guard_test.go +++ b/cmd/coldkeep/thin_cli_dependency_guard_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "testing" ) @@ -24,23 +25,42 @@ func TestProductionCLIUsesOnlyApplicationAndEngineExecutionBoundaries(t *testing } allowed := map[string]bool{ - "application_session.go:openApplicationSession:application.Open": true, - "init.go:initCommand:storage.IsRegisteredCompressionCodec": true, - "main.go:resolveTraceOptions:observability.NewJSONTraceSink": true, - "main.go:newSimulationObservabilityService:observability.NewService": true, - "main.go:captureBenchmarkDiagnosticFinalState:db.BuildPostgresConnStringFromEnv": true, - "main.go:createTemporaryBenchmarkDatabase:db.BuildPostgresConnStringFromEnv": true, - "main.go:captureBenchmarkState:db.BuildPostgresConnStringFromEnv": true, - "main.go:runSimulateCommand:storage.ParseStorageContext": true, - "main.go:runSimulateCommand:storage.StoreFileWithStorageContext": true, - "main.go:runSimulateCommand:storage.StoreFileWithStorageContextAndCodec": true, - "main.go:runSimulateCommand:storage.StoreFolderWithStorageContextAndOptions": true, - "main.go:runSimulateCommand:storage.StoreFolderWithStorageContextAndCodecAndOptions": true, - "main.go:parseSnapshotQuery:snapshot.NormalizeSnapshotPath": true, - "main.go:parseSnapshotRestoreExactPathSelectors:snapshot.NormalizeSnapshotPath": true, - "main.go:parseSnapshotRestorePrefixSelectors:snapshot.NormalizeSnapshotPath": true, + "application_session.go:openApplicationSession:application.Open": true, + "init.go:initCommand:storage.IsRegisteredCompressionCodec": true, + "main.go:resolveTraceOptions:observability.NewJSONTraceSink": true, + "main.go:newSimulationObservabilityService:observability.NewService": true, + "main.go:captureBenchmarkDiagnosticFinalState:db.BuildPostgresConnStringFromEnv": true, + "main.go:createTemporaryBenchmarkDatabase:db.BuildPostgresConnStringFromEnv": true, + "main.go:captureBenchmarkState:db.BuildPostgresConnStringFromEnv": true, + "main.go:buildBenchmarkDiagnosticFinalState:internalgc.BuildPlan": true, + "main.go:buildBenchmarkDiagnosticFinalState:verify.CheckPhysicalFileGraphIntegrity": true, + "main.go:buildBenchmarkDiagnosticFinalState:verify.CheckSnapshotReachabilityIntegrity": true, + "main.go:runSimulateCommand:storage.ParseStorageContext": true, + "main.go:runSimulateCommand:storage.StoreFileWithStorageContext": true, + "main.go:runSimulateCommand:storage.StoreFileWithStorageContextAndCodec": true, + "main.go:runSimulateCommand:storage.StoreFolderWithStorageContextAndOptions": true, + "main.go:runSimulateCommand:storage.StoreFolderWithStorageContextAndCodecAndOptions": true, + "main.go:parseSnapshotQuery:snapshot.NormalizeSnapshotPath": true, + "main.go:parseSnapshotRestoreExactPathSelectors:snapshot.NormalizeSnapshotPath": true, + "main.go:parseSnapshotRestorePrefixSelectors:snapshot.NormalizeSnapshotPath": true, + } + allowedImports := map[string]bool{ + "init.go:github.com/franchoy/coldkeep/internal/storage": true, + "init.go:github.com/franchoy/coldkeep/internal/storage/compression": true, + "main.go:github.com/franchoy/coldkeep/internal/db": true, + "main.go:github.com/franchoy/coldkeep/internal/gc": true, + "main.go:github.com/franchoy/coldkeep/internal/maintenance": true, + "main.go:github.com/franchoy/coldkeep/internal/observability": true, + "main.go:github.com/franchoy/coldkeep/internal/recovery": true, + "main.go:github.com/franchoy/coldkeep/internal/snapshot": true, + "main.go:github.com/franchoy/coldkeep/internal/storage": true, + "main.go:github.com/franchoy/coldkeep/internal/verify": true, + "observability_engine_adapters.go:github.com/franchoy/coldkeep/internal/observability": true, + "stored_path_engine_adapters.go:github.com/franchoy/coldkeep/internal/recovery": true, + "stored_path_engine_adapters.go:github.com/franchoy/coldkeep/internal/storage": true, } used := make(map[string]bool) + usedImports := make(map[string]bool) var violations []string for _, entry := range entries { name := entry.Name() @@ -51,6 +71,21 @@ func TestProductionCLIUsesOnlyApplicationAndEngineExecutionBoundaries(t *testing if err != nil { t.Fatalf("parse %s: %v", name, err) } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("parse import in %s: %v", name, err) + } + if !isGuardedCLIImport(path) { + continue + } + key := name + ":" + path + if allowedImports[key] { + usedImports[key] = true + continue + } + violations = append(violations, "unreviewed lower-layer import: "+key) + } inspectCalls := func(owner string, body ast.Node) { ast.Inspect(body, func(node ast.Node) bool { call, ok := node.(*ast.CallExpr) @@ -109,6 +144,11 @@ func TestProductionCLIUsesOnlyApplicationAndEngineExecutionBoundaries(t *testing violations = append(violations, "stale allowlist entry: "+exception) } } + for exception := range allowedImports { + if !usedImports[exception] { + violations = append(violations, "stale import allowlist entry: "+exception) + } + } sort.Strings(violations) if len(violations) > 0 { t.Fatalf("production CLI bypasses application/engine boundary:\n %s", strings.Join(violations, "\n ")) @@ -117,13 +157,32 @@ func TestProductionCLIUsesOnlyApplicationAndEngineExecutionBoundaries(t *testing func isGuardedCLIPackage(name string) bool { switch name { - case "application", "catalog", "db", "maintenance", "observability", "recovery", "snapshot", "storage": + case "application", "catalog", "db", "internalgc", "maintenance", "observability", "recovery", "snapshot", "storage", "verify": return true default: return false } } +func isGuardedCLIImport(path string) bool { + for _, prefix := range []string{ + "github.com/franchoy/coldkeep/internal/catalog", + "github.com/franchoy/coldkeep/internal/db", + "github.com/franchoy/coldkeep/internal/gc", + "github.com/franchoy/coldkeep/internal/maintenance", + "github.com/franchoy/coldkeep/internal/observability", + "github.com/franchoy/coldkeep/internal/recovery", + "github.com/franchoy/coldkeep/internal/snapshot", + "github.com/franchoy/coldkeep/internal/storage", + "github.com/franchoy/coldkeep/internal/verify", + } { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} + func isProjectionOnlyCall(filename, packageName, selector string) bool { if filename == "observability_engine_adapters.go" && packageName == "observability" { return true diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 3333d4cf..9b631cd3 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–18 are complete; Phase 19 is next. + 0–19 are complete; Phase 20 is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index ac562996..93c1e398 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -123,11 +123,13 @@ ## Phase 19 — Dependency and Contract Hardening -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 19 dependency and contract hardening](v1.13.12-phase19-dependency-and-contract-hardening.md). ## Phase 20 — Compatibility and Adversarial Regression -**Status:** Not started +**Status:** Next ## Phase 21 — Exact-Head Candidate Release Gate diff --git a/docs/release/v1.13/v1.13.12-phase19-dependency-and-contract-hardening.md b/docs/release/v1.13/v1.13.12-phase19-dependency-and-contract-hardening.md new file mode 100644 index 00000000..2fa47065 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase19-dependency-and-contract-hardening.md @@ -0,0 +1,94 @@ +# Coldkeep v1.13.12 Phase 19 — Dependency and Contract Hardening + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `19 — Dependency and Contract Hardening` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Contract closure + +The exact 25-method `Engine` operation set and 13-method aggregate `Catalog` +operation set are now executable inventories. A removal, rename, or unreviewed +addition fails the contract suite. Reflection coverage is coupled to those +interfaces so every request/result root must participate in recursive +neutrality validation. + +Catalog neutrality now covers current-file queries, repository configuration, +graph nodes, and the typed catalog error in addition to planning contracts. +Catalog error causes are private while remaining available through +`errors.Is` and `errors.As`; invalid catalog error codes fail closed to +`operation_failed`. + +Every public `DefaultEngine` operation begins with a statically enforced +`TranslateError` boundary. Existing typed classifications remain unchanged, +while previously raw operation errors now carry stable operation/code context +without changing their human message or unwrap chain. Request validation for +verify, snapshot show/diff/create/delete/restore, store/folder store, +remove/restore, and stored-path restore receives explicit `invalid_argument` +classification; missing snapshot show receives `not_found`. + +The transitional engine and catalog `ErrNotImplemented` sentinels and catalog +deferred classifier are removed. `unsupported` remains a valid stable typed +code for real capability errors, but no promised v1.x operation advertises or +returns a not-implemented result. A production-source guard prevents that +surface from returning. + +## Dependency enforcement + +The CLI AST guard now freezes every direct lower-layer import as well as every +lower-layer execution call. Each exception is tied to an exact file and use: +presentation/compatibility projection, init-time codec validation, isolated +simulation, or benchmark/determinism diagnostics. New and stale import or call +allowlist entries both fail. + +The benchmark-only GC and physical/snapshot verification observers are now +explicit exceptions rather than uninspected aliases. Normal production +repository commands retain only application/engine execution. The application +package may directly import only engine, configured DB, and storage adapters, +preventing it from becoming a second business-orchestration layer. + +## Direct dual-backend proof + +The shared engine matrix now directly covers `ListFiles`, `SearchFiles`, +`GetConfiguration`, and `SetConfiguration` on SQLite and PostgreSQL, including +deterministic ordering, filters, changed/unchanged results, normalization, and +cancellation. Together with the existing read, mutation, planning, repair, +recovery, Doctor, snapshot-selector, and catalog suites, every active operation +family has direct headless and dual-backend evidence. + +The first comprehensive PostgreSQL run exposed a test-fixture-only mismatch: +the reused current-file seeder inserted integer `1` into PostgreSQL's Boolean +`is_metadata_complete` column. All scratch databases were dropped. Binding the +neutral Boolean value `true` fixed the fixture without changing production +code, and the complete catalog/engine matrix then passed. + +The retained verbose logs record every exact created and dropped name: + +- first attempt catalog group: suffix `377dc6`, indices `1`–`b`; +- first attempt engine group: suffix `377dec`, indices `1`–`15`; +- passing attempt catalog group: suffix `378267`, indices `1`–`b`; +- passing attempt engine group: suffix `3782d0`, indices `1`–`15`. + +Each group contains the exact test-derived prefix emitted by `backendtest`. +Created/drop entries are paired one-for-one. A final read-only maintenance +query over all four exact run identifiers returned no database names; no +ordinary database was reset, migrated, truncated, or dropped. + +## Validation + +- Exact operation inventories, neutrality walks, typed-boundary source guard, + no-deferred-surface guard, dependency direction, and thin-CLI AST/import + guards: pass. +- Complete catalog and engine SQLite/PostgreSQL contract matrix: pass. +- Full isolated `go test -count=1 ./...`: pass. +- Focused catalog/engine/application/CLI race matrix: pass. +- Release-state validator and diff guards: pass. +- Validation root: `/tmp/coldkeep-v11312-phase19.ZSxtzo/`. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. + +Phase 20 is authorized to execute the frozen compatibility, adversarial, +repository-format, storage-layout, snapshot, repair, recovery, GC, and +coordination regression matrix. Phase 19 does not authorize candidate freeze, +merge, publication, v1.x closure, or v2.x handoff. diff --git a/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md b/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md index f4f7f879..251bb74d 100644 --- a/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md +++ b/docs/release/v1.13/v1.13.12-phase2-engine-error-and-neutrality-foundation.md @@ -13,13 +13,16 @@ conflict, cancellation, and ordinary operation failure. `engine.Error` exports only neutral scalar classification fields; its wrapped cause is private and remains available through the standard Go error chain. -`TranslateError` deterministically recognizes cancellation/deadline, -`ErrNotImplemented`, and typed invariant errors while preserving exact human +At this foundation phase, `TranslateError` recognized cancellation/deadline, +the transitional `ErrNotImplemented`, and typed invariant errors while preserving exact human messages. `TranslateErrorAs` permits operation-specific semantic mapping without overriding universal cancellation or invariant classification. `CodeOf`, `IsCode`, and the compatibility `IsUnsupported` helper provide stable inspection without string parsing. +Phase 19 removed the transitional sentinel after every promised operation was +active; `unsupported` now exists only as the stable typed code. + No active operation was rerouted and no CLI error, JSON, exit, schema, storage, or backend behavior changed in this phase. diff --git a/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md b/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md index 9d53cf5a..281723c4 100644 --- a/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md +++ b/docs/release/v1.13/v1.13.12-phase4-catalog-contract-replacement.md @@ -8,9 +8,10 @@ ## Contract replacement The four provisional planning shapes are replaced with complete neutral -contracts. This phase defines the contracts and their invariants only; the -four service methods retain the typed transitional `ErrNotImplemented` result -until their implementation phases. +contracts. This phase defined the contracts and their invariants before the +four service methods were activated in Phases 5–9. The transitional +`ErrNotImplemented` result used at this phase was removed in Phase 19 after all +promised operations were production-consumed. - Snapshot graph results contain deterministically ordered nodes, roots, child relations, and an explicit none/present/missing parent state. A missing diff --git a/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md b/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md index d6c2e040..5ea78a5f 100644 --- a/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md +++ b/docs/release/v1.13/v1.13.12-phase9-gc-plan-implementation-and-adoption.md @@ -33,8 +33,8 @@ packed-block liveness, deletion transactions, rollback, dry-run behavior, failure order, and reporting are unchanged. All four promised catalog planning APIs are now implemented and production -consumed. The catalog deferred sentinel remains only as historical error -taxonomy compatibility; no active catalog method returns it. +consumed. Phase 19 subsequently removed the transitional catalog sentinel and +its deferred taxonomy after confirming that no active method returned it. ## Validation diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index 51e3f1ec..fb187b15 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,9 +11,10 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–18 are Complete. Phase 19 is Next: harden dependency direction and -neutral contracts, run the comprehensive engine/catalog dual-backend matrix, -and remove remaining promised-but-unimplemented or stale surfaces. +Phases 0–19 are Complete. Phase 20 is Next: run the frozen compatibility and +adversarial regression matrix and prove that the architecture completion did +not change public, storage, database, lock, transaction, or failure-order +semantics. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index 676d69a3..fd39c1d4 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -293,11 +293,28 @@ ## Phase 19 — Dependency and Contract Hardening -**Phase status:** Next +**Phase status:** Complete + +- [x] Exact Engine and Catalog operation inventories fail on silent removal or + unreviewed additions. +- [x] Recursive reflection coverage includes every active request/result graph, + current-file/configuration DTOs, catalog errors, and nested neutral values. +- [x] Every public Engine method begins with an enforced typed-error boundary; + causes and exact messages remain compatible while operation/code context is + always available. +- [x] Transitional Engine/Catalog `ErrNotImplemented` and deferred-taxonomy + surfaces are removed; a source guard prevents their return. +- [x] CLI lower-layer imports and calls are exact allowlisted with stale-entry + detection; application composition imports only engine, DB, and storage + adapters. +- [x] List/search/configuration direct engine contracts now run on both SQLite + and PostgreSQL, and the complete catalog/engine dual-backend matrix passes. +- [x] Full isolated regression, focused race, exact PostgreSQL scratch cleanup, + release-state, and clean-diff guards pass. ## Phase 20 — Compatibility and Adversarial Regression -**Phase status:** Not started +**Phase status:** Next ## Phase 21 — Exact-Head Candidate Release Gate diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 9b5e32e3..3a29a0cd 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -1,13 +1,10 @@ // Package catalog is the behavior-preserving metadata facade for coldkeep. // -// v1.12 Phase 3 — Catalog Facade Skeleton. -// // The catalog owns metadata truth: logical identity, physical-file mapping, // snapshot metadata and graph, reachability, chunk/block/container placement, -// restore-plan metadata, and GC-plan metadata. Storage owns payload bytes; -// the engine owns operation orchestration. This package introduces the catalog -// boundary WITHOUT migrating any command orchestration. No snapshot, GC, -// restore, store, remove, repair, or recovery behavior is moved here in Phase 3. +// restore-plan metadata, GC-plan metadata, current-file queries, and repository +// configuration. Storage owns payload bytes; the engine owns correctness and +// operation orchestration. // // # Invariants // diff --git a/internal/catalog/contracts_test.go b/internal/catalog/contracts_test.go index a03cbdc1..9aa79936 100644 --- a/internal/catalog/contracts_test.go +++ b/internal/catalog/contracts_test.go @@ -95,7 +95,4 @@ func TestCatalogErrorPreservesStableContextAndCause(t *testing.T) { if err.Operation != "load graph" || err.Invariant != "acyclic_snapshot_graph" { t.Fatalf("typed context lost: %+v", err) } - if !catalog.IsCode(catalog.ErrNotImplemented, catalog.ErrorUnsupported) || !catalog.IsDeferred(catalog.ErrNotImplemented) { - t.Fatalf("deferred sentinel not typed: %v", catalog.ErrNotImplemented) - } } diff --git a/internal/catalog/errors.go b/internal/catalog/errors.go index 39a15a65..289e4ab6 100644 --- a/internal/catalog/errors.go +++ b/internal/catalog/errors.go @@ -23,7 +23,7 @@ type Error struct { Operation string Invariant string Message string - Cause error + cause error } func (e *Error) Error() string { @@ -31,8 +31,8 @@ func (e *Error) Error() string { return "" } message := e.Message - if message == "" && e.Cause != nil { - message = e.Cause.Error() + if message == "" && e.cause != nil { + message = e.cause.Error() } if e.Operation == "" { return message @@ -47,11 +47,14 @@ func (e *Error) Unwrap() error { if e == nil { return nil } - return e.Cause + return e.cause } func NewError(code ErrorCode, operation, invariant, message string, cause error) *Error { - return &Error{Code: code, Operation: operation, Invariant: invariant, Message: message, Cause: cause} + if !validErrorCode(code) { + code = ErrorOperationFailed + } + return &Error{Code: code, Operation: operation, Invariant: invariant, Message: message, cause: cause} } func CodeOf(err error) (ErrorCode, bool) { @@ -67,17 +70,17 @@ func IsCode(err error, code ErrorCode) bool { return ok && actual == code } -var deferredCause = errors.New("catalog operation not implemented") - -// ErrNotImplemented is retained for error-taxonomy compatibility. No active -// catalog operation returns it after Phase 9 completed all four planning APIs. -var ErrNotImplemented error = NewError( - ErrorUnsupported, - "deferred planning operation", - "catalog_planning_api_must_be_implemented", - deferredCause.Error(), - deferredCause, -) - -// IsDeferred recognizes only the transitional sentinel, not all unsupported errors. -func IsDeferred(err error) bool { return errors.Is(err, ErrNotImplemented) } +func validErrorCode(code ErrorCode) bool { + switch code { + case ErrorInvalidArgument, + ErrorNotFound, + ErrorUnsupported, + ErrorInvariantViolation, + ErrorConflict, + ErrorCancelled, + ErrorOperationFailed: + return true + default: + return false + } +} diff --git a/internal/catalog/errors_test.go b/internal/catalog/errors_test.go index 4c17bd0c..3c70154b 100644 --- a/internal/catalog/errors_test.go +++ b/internal/catalog/errors_test.go @@ -2,50 +2,22 @@ package catalog_test import ( "errors" - "fmt" "testing" "github.com/franchoy/coldkeep/internal/catalog" - "github.com/franchoy/coldkeep/internal/engine" ) -func TestErrNotImplementedRemainsCatalogDeferredSentinel(t *testing.T) { - if !errors.Is(catalog.ErrNotImplemented, catalog.ErrNotImplemented) { - t.Fatal("expected catalog.ErrNotImplemented to remain errors.Is compatible with itself") - } - if !catalog.IsDeferred(catalog.ErrNotImplemented) { - t.Fatal("expected catalog.ErrNotImplemented to classify as deferred") - } -} - -func TestIsDeferredRecognizesWrappedErrNotImplemented(t *testing.T) { - err := fmt.Errorf("wrapped catalog deferred method: %w", catalog.ErrNotImplemented) - if !errors.Is(err, catalog.ErrNotImplemented) { - t.Fatalf("expected wrapped error to remain catalog.ErrNotImplemented-compatible, got %v", err) - } - if !catalog.IsDeferred(err) { - t.Fatalf("expected wrapped catalog.ErrNotImplemented to classify as deferred, got %v", err) +func TestCatalogErrorCauseIsPrivateButPreservesErrorsIs(t *testing.T) { + cause := errors.New("backend unavailable") + err := catalog.NewError(catalog.ErrorOperationFailed, "load metadata", "", "metadata load failed", cause) + if !errors.Is(err, cause) { + t.Fatalf("catalog error lost private cause: %v", err) } } -func TestIsDeferredRejectsUnrelatedErrors(t *testing.T) { - for _, err := range []error{ - nil, - errors.New("plain unrelated error"), - fmt.Errorf("wrapped unrelated: %w", errors.New("other")), - fmt.Errorf("catalog: list snapshots: query failed"), - } { - if catalog.IsDeferred(err) { - t.Fatalf("expected unrelated error to not classify as deferred: %v", err) - } - } -} - -func TestCatalogAndEngineErrNotImplementedRemainDistinct(t *testing.T) { - if catalog.IsDeferred(engine.ErrNotImplemented) { - t.Fatalf("expected engine.ErrNotImplemented to not classify as catalog deferred") - } - if engine.IsUnsupported(catalog.ErrNotImplemented) { - t.Fatalf("expected catalog.ErrNotImplemented to not classify as engine unsupported") +func TestCatalogErrorRejectsUnknownCodeFailClosed(t *testing.T) { + err := catalog.NewError(catalog.ErrorCode("invented"), "load metadata", "", "metadata load failed", nil) + if err.Code != catalog.ErrorOperationFailed { + t.Fatalf("unknown code should fail closed: %+v", err) } } diff --git a/internal/catalog/neutrality_test.go b/internal/catalog/neutrality_test.go index 451f0eb6..21f61b95 100644 --- a/internal/catalog/neutrality_test.go +++ b/internal/catalog/neutrality_test.go @@ -2,7 +2,6 @@ package catalog_test import ( "reflect" - "strings" "testing" "github.com/franchoy/coldkeep/internal/catalog" @@ -12,32 +11,85 @@ import ( // graph. It rejects interface/any fields and dependencies on database, storage, // domain, CLI, renderer, and stream implementations. func TestCatalogContractGraphsAreNeutral(t *testing.T) { - roots := []any{ + for _, root := range allCatalogContractRoots() { + seen := make(map[reflect.Type]bool) + checkCatalogContractType(t, reflect.TypeOf(root), reflect.TypeOf(root).Name(), seen) + } +} + +func allCatalogContractRoots() []any { + return []any{ + catalog.Error{}, catalog.LogicalFileRef{}, catalog.PhysicalFileRef{}, + catalog.CurrentFileRef{}, catalog.CurrentFilePage{}, catalog.CurrentFileSearch{}, + catalog.RepositoryConfigurationRef{}, catalog.SetRepositoryConfigurationResult{}, catalog.SnapshotRef{}, catalog.SnapshotFilter{}, - catalog.SnapshotGraph{}, catalog.ReachabilityRoots{}, + catalog.SnapshotGraphNode{}, catalog.SnapshotGraph{}, catalog.ReachabilityRoots{}, catalog.ContainerPlacementRef{}, catalog.LegacyChunkPlacement{}, catalog.PackedChunkPlacement{}, catalog.ChunkPlacementRef{}, catalog.RestorePlanInput{}, catalog.RestoreLogicalFileRef{}, catalog.RestoreSourceRef{}, catalog.RestorePlanMetadata{}, catalog.GCPlanInput{}, catalog.GCReachabilityRoot{}, catalog.GCPlanMetadata{}, } +} - forbiddenPackages := []string{ - "database/sql", "internal/storage", "internal/domain", "internal/db", - "cmd/coldkeep", "internal/cli", "github.com/spf13/cobra", "io", +func TestCatalogOperationSetAndNeutralityCoverageAreComplete(t *testing.T) { + wantMethods := []string{ + "FindLogicalFile", + "FindPhysicalFilesForLogicalFile", + "FindSnapshot", + "GetRepositoryConfiguration", + "ListCurrentFiles", + "ListSnapshots", + "LoadChunkPlacements", + "LoadGCPlanMetadata", + "LoadReachabilityRoots", + "LoadRestorePlanMetadata", + "LoadSnapshotGraph", + "SearchCurrentFiles", + "SetRepositoryConfiguration", } - for _, root := range roots { - seen := make(map[reflect.Type]bool) - checkCatalogContractType(t, reflect.TypeOf(root), reflect.TypeOf(root).Name(), forbiddenPackages, seen) + covered := make(map[reflect.Type]bool) + for _, root := range allCatalogContractRoots() { + covered[reflect.TypeOf(root)] = true + } + + interfaceType := reflect.TypeOf((*catalog.Catalog)(nil)).Elem() + if interfaceType.NumMethod() != len(wantMethods) { + t.Fatalf("catalog operation count changed: got=%d want=%d", interfaceType.NumMethod(), len(wantMethods)) + } + for i, want := range wantMethods { + method := interfaceType.Method(i) + if method.Name != want { + t.Errorf("catalog operation %d changed: got=%q want=%q", i, method.Name, want) + } + for arg := 0; arg < method.Type.NumIn(); arg++ { + assertCatalogMethodContractCovered(t, method.Name, method.Type.In(arg), covered) + } + for result := 0; result < method.Type.NumOut(); result++ { + assertCatalogMethodContractCovered(t, method.Name, method.Type.Out(result), covered) + } } } -func checkCatalogContractType(t *testing.T, typ reflect.Type, path string, forbiddenPackages []string, seen map[reflect.Type]bool) { +func assertCatalogMethodContractCovered(t *testing.T, method string, typ reflect.Type, covered map[reflect.Type]bool) { + t.Helper() + for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice || typ.Kind() == reflect.Array { + typ = typ.Elem() + } + if typ.PkgPath() != "github.com/franchoy/coldkeep/internal/catalog" || typ.Kind() != reflect.Struct { + return + } + if !covered[typ] { + t.Errorf("Catalog.%s contract %s is absent from the neutrality walk", method, typ) + } +} + +func checkCatalogContractType(t *testing.T, typ reflect.Type, path string, seen map[reflect.Type]bool) { t.Helper() for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice || typ.Kind() == reflect.Array || typ.Kind() == reflect.Map { if typ.Kind() == reflect.Map { - checkCatalogContractType(t, typ.Key(), path+".", forbiddenPackages, seen) + checkCatalogContractType(t, typ.Key(), path+".", seen) } typ = typ.Elem() } @@ -54,11 +106,9 @@ func checkCatalogContractType(t *testing.T, typ reflect.Type, path string, forbi t.Errorf("%s exposes executable or unsafe type %v", path, typ) return } - for _, forbidden := range forbiddenPackages { - if typ.PkgPath() == forbidden || strings.Contains(typ.PkgPath(), forbidden) { - t.Errorf("%s exposes forbidden package type %v", path, typ) - return - } + if pkg := typ.PkgPath(); pkg != "" && pkg != "time" && pkg != "github.com/franchoy/coldkeep/internal/catalog" { + t.Errorf("%s exposes non-neutral package type %v from %q", path, typ, pkg) + return } if typ.Kind() != reflect.Struct || (typ.PkgPath() != "" && typ.PkgPath() != "github.com/franchoy/coldkeep/internal/catalog") { return @@ -68,6 +118,6 @@ func checkCatalogContractType(t *testing.T, typ reflect.Type, path string, forbi if !field.IsExported() { continue } - checkCatalogContractType(t, field.Type, path+"."+field.Name, forbiddenPackages, seen) + checkCatalogContractType(t, field.Type, path+"."+field.Name, seen) } } diff --git a/internal/engine/contracts_test.go b/internal/engine/contracts_test.go index 2fc76130..7409c675 100644 --- a/internal/engine/contracts_test.go +++ b/internal/engine/contracts_test.go @@ -1,7 +1,14 @@ package engine_test import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" "reflect" + "runtime" + "strings" "testing" "time" @@ -119,6 +126,145 @@ func TestActiveEngineContractNeutralityCoverage(t *testing.T) { } } +func TestEngineOperationSetIsComplete(t *testing.T) { + want := []string{ + "Doctor", + "GarbageCollect", + "GetConfiguration", + "Inspect", + "ListFiles", + "PlanGarbageCollection", + "Recover", + "Remove", + "RemoveStoredPaths", + "Repair", + "Restore", + "RestoreStoredPath", + "SearchFiles", + "SetConfiguration", + "SnapshotCreate", + "SnapshotDelete", + "SnapshotDiff", + "SnapshotList", + "SnapshotRestore", + "SnapshotShow", + "SnapshotStats", + "Stats", + "Store", + "StoreFolder", + "Verify", + } + interfaceType := reflect.TypeOf((*engine.Engine)(nil)).Elem() + if interfaceType.NumMethod() != len(want) { + t.Fatalf("engine operation count changed: got=%d want=%d", interfaceType.NumMethod(), len(want)) + } + for i, name := range want { + if got := interfaceType.Method(i).Name; got != name { + t.Errorf("engine operation %d changed: got=%q want=%q", i, got, name) + } + } +} + +func TestPromisedProductionSurfacesContainNoDeferredSentinels(t *testing.T) { + _, current, _, _ := runtime.Caller(0) + for _, dir := range []string{filepath.Dir(current), filepath.Join(filepath.Dir(current), "..", "catalog")} { + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + path := filepath.Join(dir, entry.Name()) + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"ErrNotImplemented", "operation not implemented"} { + if strings.Contains(string(source), forbidden) { + t.Errorf("promised production surface retains %q in %s", forbidden, path) + } + } + } + } +} + +func TestEveryEngineOperationHasTypedErrorBoundary(t *testing.T) { + interfaceType := reflect.TypeOf((*engine.Engine)(nil)).Elem() + want := make(map[string]bool, interfaceType.NumMethod()) + for i := 0; i < interfaceType.NumMethod(); i++ { + want[interfaceType.Method(i).Name] = true + } + + _, current, _, _ := runtime.Caller(0) + dir := filepath.Dir(current) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + seen := make(map[string]bool, len(want)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + path := filepath.Join(dir, entry.Name()) + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + for _, declaration := range file.Decls { + method, ok := declaration.(*ast.FuncDecl) + if !ok || !want[method.Name.Name] || !hasDefaultEngineReceiver(method) { + continue + } + seen[method.Name.Name] = true + if method.Body == nil || len(method.Body.List) == 0 { + t.Errorf("Engine.%s has no typed error boundary", method.Name.Name) + continue + } + boundary, ok := method.Body.List[0].(*ast.DeferStmt) + if !ok || !deferCallsTranslateError(boundary) { + t.Errorf("Engine.%s must begin with a deferred TranslateError boundary", method.Name.Name) + } + } + } + for method := range want { + if !seen[method] { + t.Errorf("Engine.%s implementation was not inspected", method) + } + } +} + +func hasDefaultEngineReceiver(method *ast.FuncDecl) bool { + if method.Recv == nil || len(method.Recv.List) != 1 { + return false + } + receiver, ok := method.Recv.List[0].Type.(*ast.StarExpr) + if !ok { + return false + } + name, ok := receiver.X.(*ast.Ident) + return ok && name.Name == "DefaultEngine" +} + +func deferCallsTranslateError(statement *ast.DeferStmt) bool { + found := false + ast.Inspect(statement, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + name, ok := call.Fun.(*ast.Ident) + if ok && name.Name == "TranslateError" { + found = true + return false + } + return true + }) + return found +} + func activeSnapshotMutationContractTypes() []struct { name string val any diff --git a/internal/engine/current_configuration_backend_contract_test.go b/internal/engine/current_configuration_backend_contract_test.go new file mode 100644 index 00000000..b3a1fae7 --- /dev/null +++ b/internal/engine/current_configuration_backend_contract_test.go @@ -0,0 +1,87 @@ +package engine_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/franchoy/coldkeep/internal/engine" + filestate "github.com/franchoy/coldkeep/internal/status" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" +) + +func TestEngineCurrentFilesAndConfigurationAcrossBackends(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{}, func(t *testing.T, backend backendtest.Backend) { + seedEngineCurrentFile(t, backend.DB, 1901, "/phase19/report.txt", "phase19-report", 120, filestate.LogicalFileCompleted) + seedEngineCurrentFile(t, backend.DB, 1902, "/phase19/notes.txt", "phase19-notes", 40, filestate.LogicalFileCompleted) + seedEngineCurrentFile(t, backend.DB, 1903, "/phase19/aborted.txt", "phase19-aborted", 80, filestate.LogicalFileAborted) + eng, err := engine.New(engine.Config{DB: backend.DB}) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + + listed, err := eng.ListFiles(context.Background(), engine.ListFilesRequest{}) + if err != nil { + t.Fatalf("ListFiles: %v", err) + } + if got, want := currentFileNames(listed.Files), []string{"/phase19/notes.txt", "/phase19/report.txt"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ListFiles paths: got=%v want=%v", got, want) + } + + searched, err := eng.SearchFiles(context.Background(), engine.SearchFilesRequest{ + NameContains: []string{"phase19", "report"}, + MinSizeBytes: []int64{100}, + MaxSizeBytes: []int64{200}, + }) + if err != nil || len(searched.Files) != 1 || searched.Files[0].Name != "/phase19/report.txt" { + t.Fatalf("SearchFiles: got=(%+v, %v)", searched, err) + } + + set, err := eng.SetConfiguration(context.Background(), engine.SetConfigurationRequest{ + Key: engine.ConfigurationCompression, Value: " zstd ", + }) + if err != nil || !set.Changed || set.Value != "zstd" { + t.Fatalf("SetConfiguration: got=(%+v, %v)", set, err) + } + unchanged, err := eng.SetConfiguration(context.Background(), engine.SetConfigurationRequest{ + Key: engine.ConfigurationCompression, Value: "zstd", + }) + if err != nil || unchanged.Changed { + t.Fatalf("unchanged SetConfiguration: got=(%+v, %v)", unchanged, err) + } + got, err := eng.GetConfiguration(context.Background(), engine.GetConfigurationRequest{Key: engine.ConfigurationCompression}) + if err != nil || got.Value != "zstd" { + t.Fatalf("GetConfiguration: got=(%+v, %v)", got, err) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + operations := []struct { + name string + run func() error + }{ + {"list", func() error { + _, err := eng.ListFiles(cancelled, engine.ListFilesRequest{}) + return err + }}, + {"search", func() error { + _, err := eng.SearchFiles(cancelled, engine.SearchFilesRequest{NameContains: []string{"phase19"}}) + return err + }}, + {"get configuration", func() error { + _, err := eng.GetConfiguration(cancelled, engine.GetConfigurationRequest{Key: engine.ConfigurationCompression}) + return err + }}, + {"set configuration", func() error { + _, err := eng.SetConfiguration(cancelled, engine.SetConfigurationRequest{Key: engine.ConfigurationCompression, Value: "none"}) + return err + }}, + } + for _, operation := range operations { + if err := operation.run(); !errors.Is(err, context.Canceled) || !engine.IsCode(err, engine.ErrorCancelled) { + t.Errorf("cancelled %s: %v", operation.name, err) + } + } + }) +} diff --git a/internal/engine/current_files.go b/internal/engine/current_files.go index 59ae8aad..bf13afd4 100644 --- a/internal/engine/current_files.go +++ b/internal/engine/current_files.go @@ -8,7 +8,8 @@ import ( "github.com/franchoy/coldkeep/internal/catalog" ) -func (e *DefaultEngine) ListFiles(ctx context.Context, req ListFilesRequest) (ListFilesResult, error) { +func (e *DefaultEngine) ListFiles(ctx context.Context, req ListFilesRequest) (_ ListFilesResult, outErr error) { + defer func() { outErr = TranslateError("list files", outErr) }() if err := validateFileQueryPage(req.Limit, req.Offset); err != nil { return ListFilesResult{}, TranslateErrorAs("list files", ErrorInvalidArgument, err) } @@ -21,7 +22,8 @@ func (e *DefaultEngine) ListFiles(ctx context.Context, req ListFilesRequest) (Li return ListFilesResult{Files: currentFilesFromCatalog(refs)}, nil } -func (e *DefaultEngine) SearchFiles(ctx context.Context, req SearchFilesRequest) (SearchFilesResult, error) { +func (e *DefaultEngine) SearchFiles(ctx context.Context, req SearchFilesRequest) (_ SearchFilesResult, outErr error) { + defer func() { outErr = TranslateError("search files", outErr) }() if err := validateSearchFilesRequest(req); err != nil { return SearchFilesResult{}, TranslateErrorAs("search files", ErrorInvalidArgument, err) } diff --git a/internal/engine/current_files_test.go b/internal/engine/current_files_test.go index 2f0562c3..36d6c9e6 100644 --- a/internal/engine/current_files_test.go +++ b/internal/engine/current_files_test.go @@ -69,7 +69,7 @@ VALUES ($1, $2, $3, $4, $5, 1, 'v1-simple-rolling')`, id, path, size, hash, stat } if _, err := dbconn.Exec(` INSERT INTO physical_file (path, logical_file_id, is_metadata_complete) -VALUES ($1, $2, 1)`, path, id); err != nil { +VALUES ($1, $2, $3)`, path, id, true); err != nil { t.Fatalf("insert physical file: %v", err) } } diff --git a/internal/engine/default_engine.go b/internal/engine/default_engine.go index bf7334a2..42b2525d 100644 --- a/internal/engine/default_engine.go +++ b/internal/engine/default_engine.go @@ -86,7 +86,8 @@ func secureSnapshotIDGenerator() (string, error) { return "snap-" + hex.EncodeToString(b), nil } -func (e *DefaultEngine) Stats(ctx context.Context, req StatsRequest) (StatsResult, error) { +func (e *DefaultEngine) Stats(ctx context.Context, req StatsRequest) (_ StatsResult, outErr error) { + defer func() { outErr = TranslateError("stats", outErr) }() trace, collector := traceOptions(req.IncludeTrace) r, err := e.obs.Stats(ctx, observability.StatsOptions{ IncludeContainers: req.IncludeContainers, @@ -102,7 +103,8 @@ func (e *DefaultEngine) Stats(ctx context.Context, req StatsRequest) (StatsResul return result, nil } -func (e *DefaultEngine) Inspect(ctx context.Context, req InspectRequest) (InspectResult, error) { +func (e *DefaultEngine) Inspect(ctx context.Context, req InspectRequest) (_ InspectResult, outErr error) { + defer func() { outErr = TranslateError("inspect", outErr) }() if err := validateInspectRequest(req); err != nil { return InspectResult{}, TranslateErrorAs("inspect", ErrorInvalidArgument, err) } @@ -127,20 +129,21 @@ func (e *DefaultEngine) Inspect(ctx context.Context, req InspectRequest) (Inspec return result, nil } -func (e *DefaultEngine) Verify(ctx context.Context, req VerifyRequest) (VerifyResult, error) { +func (e *DefaultEngine) Verify(ctx context.Context, req VerifyRequest) (_ VerifyResult, outErr error) { + defer func() { outErr = TranslateError("verify", outErr) }() if err := ctx.Err(); err != nil { return VerifyResult{}, err } level, err := verifyLevelFromString(req.Level) if err != nil { - return VerifyResult{}, err + return VerifyResult{}, TranslateErrorAs("verify", ErrorInvalidArgument, err) } target := req.Target if target == "" { target = "system" } if err := validateVerifyRequest(target, req.FileID); err != nil { - return VerifyResult{}, err + return VerifyResult{}, TranslateErrorAs("verify", ErrorInvalidArgument, err) } containerDir := e.config.ContainerDir if containerDir == "" { @@ -217,7 +220,8 @@ func verifyLevelFromString(s string) (verify.VerifyLevel, error) { } } -func (e *DefaultEngine) SnapshotList(ctx context.Context, req SnapshotListRequest) (SnapshotListResult, error) { +func (e *DefaultEngine) SnapshotList(ctx context.Context, req SnapshotListRequest) (_ SnapshotListResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_list", outErr) }() var tx *sql.Tx catalogDB := catalog.DB(e.config.DB) if req.Tree { @@ -318,14 +322,18 @@ func projectSelectedSnapshotGraph(graph *catalog.SnapshotGraph, selected []catal return result, metas } -func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowRequest) (SnapshotShowResult, error) { +func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowRequest) (_ SnapshotShowResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_show", outErr) }() + if strings.TrimSpace(req.SnapshotID) == "" { + return SnapshotShowResult{}, TranslateErrorAs("snapshot_show", ErrorInvalidArgument, fmt.Errorf("snapshot id cannot be empty")) + } svc := catalog.NewServiceFromSQL(e.config.DB) ref, err := svc.FindSnapshot(ctx, req.SnapshotID) if err != nil { return SnapshotShowResult{}, err } if ref == nil { - return SnapshotShowResult{}, fmt.Errorf("snapshot %q not found", req.SnapshotID) + return SnapshotShowResult{}, TranslateErrorAs("snapshot_show", ErrorNotFound, fmt.Errorf("snapshot %q not found", req.SnapshotID)) } meta := SnapshotMeta{ ID: ref.ID, @@ -339,7 +347,7 @@ func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowReques var err error snapshotQ, err = engineQueryToSnapshotQuery(req.Query) if err != nil { - return SnapshotShowResult{}, err + return SnapshotShowResult{}, TranslateErrorAs("snapshot_show", ErrorInvalidArgument, err) } } entries, err := snapshot.ListSnapshotFiles(ctx, e.config.DB, req.SnapshotID, req.Query.Limit, snapshotQ) @@ -368,7 +376,8 @@ func (e *DefaultEngine) SnapshotShow(ctx context.Context, req SnapshotShowReques }, nil } -func (e *DefaultEngine) SnapshotStats(ctx context.Context, req SnapshotStatsRequest) (SnapshotStatsResult, error) { +func (e *DefaultEngine) SnapshotStats(ctx context.Context, req SnapshotStatsRequest) (_ SnapshotStatsResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_stats", outErr) }() stats, err := snapshot.GetSnapshotStats(ctx, e.config.DB, req.SnapshotID) if err != nil { return SnapshotStatsResult{}, err @@ -390,30 +399,45 @@ func (e *DefaultEngine) SnapshotStats(ctx context.Context, req SnapshotStatsRequ return result, nil } -func (e *DefaultEngine) SnapshotDiff(ctx context.Context, req SnapshotDiffRequest) (SnapshotDiffResult, error) { +func (e *DefaultEngine) SnapshotDiff(ctx context.Context, req SnapshotDiffRequest) (_ SnapshotDiffResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_diff", outErr) }() + if strings.TrimSpace(req.BaseID) == "" { + return SnapshotDiffResult{}, TranslateErrorAs("snapshot_diff", ErrorInvalidArgument, fmt.Errorf("base snapshot id cannot be empty")) + } + if strings.TrimSpace(req.TargetID) == "" { + return SnapshotDiffResult{}, TranslateErrorAs("snapshot_diff", ErrorInvalidArgument, fmt.Errorf("target snapshot id cannot be empty")) + } + if req.Filter != SnapshotDiffAll && req.Filter != SnapshotDiffAdded && req.Filter != SnapshotDiffRemoved && req.Filter != SnapshotDiffModified { + return SnapshotDiffResult{}, TranslateErrorAs("snapshot_diff", ErrorInvalidArgument, fmt.Errorf("unknown snapshot diff filter %q", req.Filter)) + } + if _, err := snapshotQueryOrNil(req.Query); err != nil { + return SnapshotDiffResult{}, TranslateErrorAs("snapshot_diff", ErrorInvalidArgument, err) + } if isSnapshotDiffSummaryFastPath(req) { return e.snapshotDiffSummaryFastPath(ctx, req) } return e.snapshotDiffDetailed(ctx, req) } -func (e *DefaultEngine) Remove(ctx context.Context, req RemoveRequest) (RemoveResult, error) { +func (e *DefaultEngine) Remove(ctx context.Context, req RemoveRequest) (_ RemoveResult, outErr error) { + defer func() { outErr = TranslateError("remove", outErr) }() if err := ctx.Err(); err != nil { return RemoveResult{}, err } if err := validateRemoveRequest(req); err != nil { - return RemoveResult{}, err + return RemoveResult{}, TranslateErrorAs("remove", ErrorInvalidArgument, err) } return e.removeFileIDs(req), nil } -func (e *DefaultEngine) RemoveStoredPaths(ctx context.Context, req RemoveStoredPathsRequest) (RemoveStoredPathsResult, error) { +func (e *DefaultEngine) RemoveStoredPaths(ctx context.Context, req RemoveStoredPathsRequest) (_ RemoveStoredPathsResult, outErr error) { + defer func() { outErr = TranslateError("remove_stored_paths", outErr) }() if err := ctx.Err(); err != nil { return RemoveStoredPathsResult{}, err } preflight, err := preflightRemoveStoredPaths(req) if err != nil { - return RemoveStoredPathsResult{}, err + return RemoveStoredPathsResult{}, TranslateErrorAs("remove_stored_paths", ErrorInvalidArgument, err) } if !preflight.requiresRepository { return preflight.terminalResult, nil @@ -424,12 +448,13 @@ func (e *DefaultEngine) RemoveStoredPaths(ctx context.Context, req RemoveStoredP return e.removeStoredPaths(req, preflight.prepared), nil } -func (e *DefaultEngine) Restore(ctx context.Context, req RestoreRequest) (RestoreResult, error) { +func (e *DefaultEngine) Restore(ctx context.Context, req RestoreRequest) (_ RestoreResult, outErr error) { + defer func() { outErr = TranslateError("restore", outErr) }() if err := ctx.Err(); err != nil { return RestoreResult{}, err } if err := validateRestoreRequest(req); err != nil { - return RestoreResult{}, err + return RestoreResult{}, TranslateErrorAs("restore", ErrorInvalidArgument, err) } return e.restoreFileIDs(req), nil } diff --git a/internal/engine/default_engine_routing_helpers.go b/internal/engine/default_engine_routing_helpers.go index 694fdde6..98b1caf5 100644 --- a/internal/engine/default_engine_routing_helpers.go +++ b/internal/engine/default_engine_routing_helpers.go @@ -111,7 +111,8 @@ func buildSnapshotDiffResult(req SnapshotDiffRequest, entries []SnapshotDiffEntr return res } -func (e *DefaultEngine) GarbageCollect(ctx context.Context, req GarbageCollectRequest) (GarbageCollectResult, error) { +func (e *DefaultEngine) GarbageCollect(ctx context.Context, req GarbageCollectRequest) (_ GarbageCollectResult, outErr error) { + defer func() { outErr = TranslateError("garbage_collect", outErr) }() containerDir := e.config.ContainerDir if containerDir == "" { containerDir = container.ContainersDir @@ -133,7 +134,8 @@ func (e *DefaultEngine) GarbageCollect(ctx context.Context, req GarbageCollectRe }, nil } -func (e *DefaultEngine) PlanGarbageCollection(ctx context.Context, req GarbageCollectionPlanRequest) (GarbageCollectionPlanResult, error) { +func (e *DefaultEngine) PlanGarbageCollection(ctx context.Context, req GarbageCollectionPlanRequest) (_ GarbageCollectionPlanResult, outErr error) { + defer func() { outErr = TranslateError("plan_garbage_collection", outErr) }() if ctx == nil { ctx = context.Background() } @@ -220,16 +222,22 @@ func integerValue(value any) Value { return converted } -func (e *DefaultEngine) Store(ctx context.Context, req StoreRequest) (StoreResult, error) { +func (e *DefaultEngine) Store(ctx context.Context, req StoreRequest) (_ StoreResult, outErr error) { + defer func() { outErr = TranslateError("store", outErr) }() if err := ctx.Err(); err != nil { return StoreResult{}, err } if strings.TrimSpace(req.SourcePath) == "" { - return StoreResult{}, fmt.Errorf("engine: store source path is required") + return StoreResult{}, TranslateErrorAs("store", ErrorInvalidArgument, fmt.Errorf("engine: store source path is required")) } if e.config.StoreContext == nil { return StoreResult{}, fmt.Errorf("engine: store requires injected StoreContext") } + if strings.TrimSpace(req.Codec) != "" { + if _, err := blocks.ParseCodec(req.Codec); err != nil { + return StoreResult{}, TranslateErrorAs("store", ErrorInvalidArgument, err) + } + } stored, err := storeWithOptionalCodec(*e.config.StoreContext, req) if err != nil { @@ -244,7 +252,8 @@ func (e *DefaultEngine) Store(ctx context.Context, req StoreRequest) (StoreResul }, nil } -func (e *DefaultEngine) StoreFolder(ctx context.Context, req StoreFolderRequest) (StoreFolderResult, error) { +func (e *DefaultEngine) StoreFolder(ctx context.Context, req StoreFolderRequest) (_ StoreFolderResult, outErr error) { + defer func() { outErr = TranslateError("store_folder", outErr) }() if ctx == nil { ctx = context.Background() } @@ -252,10 +261,10 @@ func (e *DefaultEngine) StoreFolder(ctx context.Context, req StoreFolderRequest) return StoreFolderResult{}, err } if strings.TrimSpace(req.SourcePath) == "" { - return StoreFolderResult{}, fmt.Errorf("engine: store folder source path is required") + return StoreFolderResult{}, TranslateErrorAs("store_folder", ErrorInvalidArgument, fmt.Errorf("engine: store folder source path is required")) } if req.Workers < 0 { - return StoreFolderResult{}, fmt.Errorf("engine: store folder workers must be zero or greater") + return StoreFolderResult{}, TranslateErrorAs("store_folder", ErrorInvalidArgument, fmt.Errorf("engine: store folder workers must be zero or greater")) } if e.config.StoreContext == nil { return StoreFolderResult{}, fmt.Errorf("engine: store folder requires injected StoreContext") @@ -270,7 +279,7 @@ func (e *DefaultEngine) StoreFolder(ctx context.Context, req StoreFolderRequest) codec, err = blocks.ParseCodec(req.Codec) } if err != nil { - return StoreFolderResult{}, err + return StoreFolderResult{}, TranslateErrorAs("store_folder", ErrorInvalidArgument, err) } stats, err := storage.StoreFolderWithStorageContextAndCodecAndOptionsWithStatsContext(ctx, *e.config.StoreContext, req.SourcePath, codec, opts) result := StoreFolderResult{SourcePath: req.SourcePath, FilesStored: stats.TotalFilesProcessed, BytesLogical: stats.TotalBytesProcessed, WorkersUsed: stats.WorkersUsed} diff --git a/internal/engine/dependency_guard_test.go b/internal/engine/dependency_guard_test.go index a3a395c0..a6067949 100644 --- a/internal/engine/dependency_guard_test.go +++ b/internal/engine/dependency_guard_test.go @@ -53,6 +53,28 @@ func TestEngineDependencyDirection(t *testing.T) { checkEngineNotDependsOnCLI(t, pkg, enginePkg, cliPkg) checkDomainNotImportsEngine(t, pkg, module, enginePkg, applicationPkg) checkCLINotImportsCatalog(t, pkg, cliPkg, catalogPkg) + checkApplicationCompositionImports(t, pkg, module, applicationPkg, enginePkg) + } +} + +// checkApplicationCompositionImports keeps application as a narrow composition +// root rather than a second orchestration layer. It may construct only the +// engine and its configured DB/storage adapters. +func checkApplicationCompositionImports(t *testing.T, pkg goListPackage, module, applicationPkg, enginePkg string) { + t.Helper() + if pkg.ImportPath != applicationPkg { + return + } + allowed := map[string]bool{ + enginePkg: true, + module + "/internal/db": true, + module + "/internal/storage": true, + } + for _, imp := range pkg.Imports { + if strings.HasPrefix(imp, module+"/internal/") && !allowed[imp] { + t.Errorf("application composition imports unapproved internal package:\n\t%s -> %s", + pkg.ImportPath, imp) + } } } diff --git a/internal/engine/doctor.go b/internal/engine/doctor.go index 326cc97e..b77c33e8 100644 --- a/internal/engine/doctor.go +++ b/internal/engine/doctor.go @@ -9,7 +9,8 @@ import ( "github.com/franchoy/coldkeep/internal/maintenance" ) -func (e *DefaultEngine) Doctor(ctx context.Context, req DoctorRequest) (DoctorResult, error) { +func (e *DefaultEngine) Doctor(ctx context.Context, req DoctorRequest) (_ DoctorResult, outErr error) { + defer func() { outErr = TranslateError("doctor", outErr) }() if ctx == nil { ctx = context.Background() } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4a90d595..6cbe625c 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -1,9 +1,10 @@ // Package engine defines the coldkeep engine boundary. // // The package owns typed operation requests and results, request-level -// validation, and behavior-preserving orchestration between CLI callers and -// lower domain packages. Storage, snapshot, verification, maintenance, and -// other domain packages retain their execution ownership. +// validation, and behavior-preserving orchestration between headless callers +// and lower domain adapters. Lower packages execute injected storage, catalog, +// verification, maintenance, and recovery details without owning the public +// operation boundary. // // # Invariants // diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 5b39a79b..fda7efd8 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -25,11 +25,6 @@ const ( ErrorOperationFailed ErrorCode = "operation_failed" ) -// ErrNotImplemented is retained while the final v1.13.12 promised surfaces -// are activated. New engine operations must return a typed ErrorUnsupported -// instead of adding another untyped unsupported sentinel. -var ErrNotImplemented = errors.New("engine operation not implemented") - // Error is the stable typed engine failure. Its exported state is deliberately // string-only and backend-neutral; cause remains private but is available to // errors.Is/errors.As through Unwrap. @@ -96,8 +91,6 @@ func TranslateError(operation string, err error) error { switch { case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): code = ErrorCancelled - case errors.Is(err, ErrNotImplemented): - code = ErrorUnsupported default: var catalogErr *catalog.Error if errors.As(err, &catalogErr) && catalogErr != nil { @@ -143,9 +136,6 @@ func TranslateErrorAs(operation string, code ErrorCode, err error) error { if universal := CodeOf(err); universal == ErrorCancelled || universal == ErrorInvariantViolation { return TranslateError(operation, err) } - if errors.Is(err, ErrNotImplemented) { - code = ErrorUnsupported - } invariantCode := "" if value, ok := invariants.Code(err); ok { invariantCode = value @@ -166,8 +156,6 @@ func CodeOf(err error) ErrorCode { switch { case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): return ErrorCancelled - case errors.Is(err, ErrNotImplemented): - return ErrorUnsupported } if _, ok := invariants.Code(err); ok { return ErrorInvariantViolation @@ -180,10 +168,9 @@ func IsCode(err error, code ErrorCode) bool { return code != "" && CodeOf(err) == code } -// IsUnsupported preserves the original sentinel classifier while recognizing -// the typed replacement used by newly activated operations. +// IsUnsupported reports the stable typed unsupported classification. func IsUnsupported(err error) bool { - return errors.Is(err, ErrNotImplemented) || IsCode(err, ErrorUnsupported) + return IsCode(err, ErrorUnsupported) } func validErrorCode(code ErrorCode) bool { diff --git a/internal/engine/errors_test.go b/internal/engine/errors_test.go index 4f035d84..08a64dfb 100644 --- a/internal/engine/errors_test.go +++ b/internal/engine/errors_test.go @@ -6,26 +6,15 @@ import ( "fmt" "testing" + "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/invariants" ) -func TestErrNotImplementedRemainsUnsupportedSentinel(t *testing.T) { - if !errors.Is(engine.ErrNotImplemented, engine.ErrNotImplemented) { - t.Fatal("expected ErrNotImplemented to remain errors.Is compatible with itself") - } - if !engine.IsUnsupported(engine.ErrNotImplemented) { - t.Fatal("expected ErrNotImplemented to classify as unsupported") - } -} - -func TestIsUnsupportedRecognizesWrappedErrNotImplemented(t *testing.T) { - err := fmt.Errorf("wrapped unsupported mode: %w", engine.ErrNotImplemented) - if !errors.Is(err, engine.ErrNotImplemented) { - t.Fatalf("expected wrapped error to remain ErrNotImplemented-compatible, got %v", err) - } +func TestIsUnsupportedRecognizesTypedError(t *testing.T) { + err := engine.NewError(engine.ErrorUnsupported, "operation", "unsupported mode", "", nil) if !engine.IsUnsupported(err) { - t.Fatalf("expected wrapped ErrNotImplemented to classify as unsupported, got %v", err) + t.Fatalf("expected typed error to classify as unsupported, got %v", err) } } @@ -95,7 +84,7 @@ func TestTranslateErrorClassifiesUniversalFailures(t *testing.T) { }{ {name: "cancelled", err: context.Canceled, want: engine.ErrorCancelled}, {name: "deadline", err: context.DeadlineExceeded, want: engine.ErrorCancelled}, - {name: "unsupported", err: fmt.Errorf("recursive: %w", engine.ErrNotImplemented), want: engine.ErrorUnsupported}, + {name: "unsupported", err: catalog.NewError(catalog.ErrorUnsupported, "operation", "", "unsupported mode", nil), want: engine.ErrorUnsupported}, {name: "invariant", err: invariant, want: engine.ErrorInvariantViolation, invariantCode: invariants.CodeGCRefusedIntegrity}, {name: "ordinary", err: errors.New("disk unavailable"), want: engine.ErrorOperationFailed}, } diff --git a/internal/engine/read_side_backend_contract_test.go b/internal/engine/read_side_backend_contract_test.go index 1c3edb9c..66b64bb1 100644 --- a/internal/engine/read_side_backend_contract_test.go +++ b/internal/engine/read_side_backend_contract_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/observability" "github.com/franchoy/coldkeep/internal/testutil/backendtest" @@ -46,7 +45,7 @@ func TestEngineReadStatsAndInspectAcrossBackends(t *testing.T) { } _, err = fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: engine.InspectPhysicalFile, EntityID: "1"}) - if !errors.Is(err, observability.ErrUnsupportedEntity) || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + if !errors.Is(err, observability.ErrUnsupportedEntity) || engine.IsUnsupported(err) { t.Fatalf("physical-file inspect classification: %v", err) } assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) @@ -106,7 +105,7 @@ func TestEngineReadVerifyAcrossBackends(t *testing.T) { t.Fatalf("seed verification inconsistency: %v", err) } _, err := fixture.engine.Verify(context.Background(), engine.VerifyRequest{Target: "system", Level: "standard"}) - if err == nil || !strings.Contains(err.Error(), "system standard verification failed") || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + if err == nil || !strings.Contains(err.Error(), "system standard verification failed") || engine.IsUnsupported(err) { t.Fatalf("Verify inconsistency classification: %v", err) } }) @@ -159,13 +158,13 @@ func TestEngineReadContextAndErrorsAcrossBackends(t *testing.T) { } }) } - if _, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "missing"}); err == nil || !strings.Contains(err.Error(), "not found") || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + if _, err := fixture.engine.SnapshotShow(context.Background(), engine.SnapshotShowRequest{SnapshotID: "missing"}); err == nil || !strings.Contains(err.Error(), "not found") || engine.IsUnsupported(err) { t.Fatalf("missing snapshot classification: %v", err) } - if _, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: "unknown", EntityID: "1"}); err == nil || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + if _, err := fixture.engine.Inspect(context.Background(), engine.InspectRequest{Entity: "unknown", EntityID: "1"}); err == nil || engine.IsUnsupported(err) { t.Fatalf("invalid inspect classification: %v", err) } - if _, err := fixture.engine.Verify(context.Background(), engine.VerifyRequest{Target: "unknown"}); err == nil || engine.IsUnsupported(err) || catalog.IsDeferred(err) { + if _, err := fixture.engine.Verify(context.Background(), engine.VerifyRequest{Target: "unknown"}); err == nil || engine.IsUnsupported(err) { t.Fatalf("invalid verify classification: %v", err) } assertEngineReadStateUnchanged(t, before, captureEngineReadState(t, backend.DB, fixture.containerDir)) diff --git a/internal/engine/read_side_taxonomy_boundary_test.go b/internal/engine/read_side_taxonomy_boundary_test.go index cdbbae17..1406b340 100644 --- a/internal/engine/read_side_taxonomy_boundary_test.go +++ b/internal/engine/read_side_taxonomy_boundary_test.go @@ -6,20 +6,19 @@ import ( "testing" "time" - "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/engine" ) -func TestReadSideStatsFailuresRemainOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestReadSideStatsFailuresRemainOutsideUnsupportedClassification(t *testing.T) { t.Run("engine construction requires db", func(t *testing.T) { _, err := engine.New(engine.Config{}) if err == nil { t.Fatal("expected engine.New to fail with nil db") } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) - t.Run("active success path stays outside unsupported and deferred", func(t *testing.T) { + t.Run("active success path stays outside unsupported", func(t *testing.T) { db := openSnapshotTestDB(t) eng, err := engine.New(engine.Config{DB: db}) if err != nil { @@ -30,10 +29,10 @@ func TestReadSideStatsFailuresRemainOutsideUnsupportedAndDeferredClassification( if err != nil { t.Fatalf("Stats: %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) - t.Run("runtime db failure stays outside unsupported and deferred", func(t *testing.T) { + t.Run("runtime db failure stays outside unsupported", func(t *testing.T) { db := openSnapshotTestDB(t) eng, err := engine.New(engine.Config{DB: db}) if err != nil { @@ -47,12 +46,12 @@ func TestReadSideStatsFailuresRemainOutsideUnsupportedAndDeferredClassification( if err == nil { t.Fatal("expected Stats to fail after db close") } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) } -func TestReadSideSnapshotShowFailuresRemainOutsideUnsupportedAndDeferredClassification(t *testing.T) { - t.Run("missing snapshot remains non-unsupported and non-deferred", func(t *testing.T) { +func TestReadSideSnapshotShowFailuresRemainOutsideUnsupportedClassification(t *testing.T) { + t.Run("missing snapshot remains non-unsupported", func(t *testing.T) { db := openSnapshotTestDB(t) eng, err := engine.New(engine.Config{DB: db}) if err != nil { @@ -66,10 +65,10 @@ func TestReadSideSnapshotShowFailuresRemainOutsideUnsupportedAndDeferredClassifi if !strings.Contains(err.Error(), "not found") { t.Fatalf("expected not found error, got %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) - t.Run("active success path stays outside unsupported and deferred", func(t *testing.T) { + t.Run("active success path stays outside unsupported", func(t *testing.T) { db := openSnapshotTestDB(t) now := time.Now().UTC().Truncate(time.Second) insertTestSnapshot(t, db, "snap-show-boundary", "full", "boundary", "", now) @@ -84,11 +83,11 @@ func TestReadSideSnapshotShowFailuresRemainOutsideUnsupportedAndDeferredClassifi if err != nil { t.Fatalf("SnapshotShow: %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) } -func TestReadSideSnapshotDiffFailuresRemainOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestReadSideSnapshotDiffFailuresRemainOutsideUnsupportedClassification(t *testing.T) { t.Run("missing base snapshot", func(t *testing.T) { db := openSnapshotTestDB(t) now := time.Now().UTC().Truncate(time.Second) @@ -109,7 +108,7 @@ func TestReadSideSnapshotDiffFailuresRemainOutsideUnsupportedAndDeferredClassifi if !strings.Contains(err.Error(), `snapshot "missing-base" not found`) { t.Fatalf("expected missing base snapshot error, got %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) t.Run("missing target snapshot", func(t *testing.T) { @@ -132,10 +131,10 @@ func TestReadSideSnapshotDiffFailuresRemainOutsideUnsupportedAndDeferredClassifi if !strings.Contains(err.Error(), `snapshot "missing-target" not found`) { t.Fatalf("expected missing target snapshot error, got %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) - t.Run("summary fast path success stays outside unsupported and deferred", func(t *testing.T) { + t.Run("summary fast path success stays outside unsupported", func(t *testing.T) { db := openSnapshotTestDB(t) now := time.Now().UTC().Truncate(time.Second) insertTestSnapshot(t, db, "snap-diff-base-boundary", "full", "", "", now) @@ -158,10 +157,10 @@ func TestReadSideSnapshotDiffFailuresRemainOutsideUnsupportedAndDeferredClassifi if err != nil { t.Fatalf("SnapshotDiff summary: %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) - t.Run("detailed success stays outside unsupported and deferred", func(t *testing.T) { + t.Run("detailed success stays outside unsupported", func(t *testing.T) { db := openSnapshotTestDB(t) seedSnapshotDiffFullFixture(t, db) @@ -177,11 +176,11 @@ func TestReadSideSnapshotDiffFailuresRemainOutsideUnsupportedAndDeferredClassifi if err != nil { t.Fatalf("SnapshotDiff detailed: %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) } -func TestReadSideSnapshotDiffValidationFailuresRemainOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestReadSideSnapshotDiffValidationFailuresRemainOutsideUnsupportedClassification(t *testing.T) { db := openSnapshotTestDB(t) eng, err := engine.New(engine.Config{DB: db}) if err != nil { @@ -199,7 +198,7 @@ func TestReadSideSnapshotDiffValidationFailuresRemainOutsideUnsupportedAndDeferr if !strings.Contains(err.Error(), "base snapshot id cannot be empty") { t.Fatalf("expected empty base id error, got %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) t.Run("empty target id", func(t *testing.T) { @@ -213,17 +212,14 @@ func TestReadSideSnapshotDiffValidationFailuresRemainOutsideUnsupportedAndDeferr if !strings.Contains(err.Error(), "target snapshot id cannot be empty") { t.Fatalf("expected empty target id error, got %v", err) } - assertReadSideNonUnsupportedAndDeferred(t, err) + assertReadSideNonUnsupported(t, err) }) } -func assertReadSideNonUnsupportedAndDeferred(t *testing.T, err error) { +func assertReadSideNonUnsupported(t *testing.T, err error) { t.Helper() if engine.IsUnsupported(err) { t.Fatalf("expected read-side error to remain outside unsupported classification: %v", err) } - if catalog.IsDeferred(err) { - t.Fatalf("expected read-side error to remain outside deferred classification: %v", err) - } } diff --git a/internal/engine/recover.go b/internal/engine/recover.go index 716e1cd5..4fe0d444 100644 --- a/internal/engine/recover.go +++ b/internal/engine/recover.go @@ -10,7 +10,8 @@ import ( // Recover executes real corrective recovery. It intentionally has no dry-run // mode: startup and explicit recovery share this exact operation. -func (e *DefaultEngine) Recover(ctx context.Context, _ RecoverRequest) (RecoverResult, error) { +func (e *DefaultEngine) Recover(ctx context.Context, _ RecoverRequest) (_ RecoverResult, outErr error) { + defer func() { outErr = TranslateError("recover", outErr) }() if ctx == nil { ctx = context.Background() } diff --git a/internal/engine/repair.go b/internal/engine/repair.go index 6e1e3a21..bc60124b 100644 --- a/internal/engine/repair.go +++ b/internal/engine/repair.go @@ -10,7 +10,8 @@ import ( "github.com/franchoy/coldkeep/internal/maintenance" ) -func (e *DefaultEngine) Repair(ctx context.Context, req RepairRequest) (RepairResult, error) { +func (e *DefaultEngine) Repair(ctx context.Context, req RepairRequest) (_ RepairResult, outErr error) { + defer func() { outErr = TranslateError("repair", outErr) }() if ctx == nil { ctx = context.Background() } diff --git a/internal/engine/repository_configuration.go b/internal/engine/repository_configuration.go index 83de2f61..0bcff52b 100644 --- a/internal/engine/repository_configuration.go +++ b/internal/engine/repository_configuration.go @@ -21,7 +21,8 @@ const ( maximumCompressionLevel = int64(9) ) -func (e *DefaultEngine) GetConfiguration(ctx context.Context, req GetConfigurationRequest) (GetConfigurationResult, error) { +func (e *DefaultEngine) GetConfiguration(ctx context.Context, req GetConfigurationRequest) (_ GetConfigurationResult, outErr error) { + defer func() { outErr = TranslateError("get configuration", outErr) }() catalogKey, err := configurationCatalogKey(req.Key) if err != nil { return GetConfigurationResult{}, TranslateErrorAs("get configuration", ErrorInvalidArgument, err) @@ -41,7 +42,8 @@ func (e *DefaultEngine) GetConfiguration(ctx context.Context, req GetConfigurati return result, nil } -func (e *DefaultEngine) SetConfiguration(ctx context.Context, req SetConfigurationRequest) (SetConfigurationResult, error) { +func (e *DefaultEngine) SetConfiguration(ctx context.Context, req SetConfigurationRequest) (_ SetConfigurationResult, outErr error) { + defer func() { outErr = TranslateError("set configuration", outErr) }() catalogKey, err := configurationCatalogKey(req.Key) if err != nil { return SetConfigurationResult{}, TranslateErrorAs("set configuration", ErrorInvalidArgument, err) diff --git a/internal/engine/restore_stored_path.go b/internal/engine/restore_stored_path.go index fa954410..f96fd1b3 100644 --- a/internal/engine/restore_stored_path.go +++ b/internal/engine/restore_stored_path.go @@ -145,14 +145,15 @@ func storageRestoreTrustedRoot(req RestoreStoredPathRequest) string { return "" } -func (e *DefaultEngine) RestoreStoredPath(ctx context.Context, req RestoreStoredPathRequest) (RestoreStoredPathResult, error) { +func (e *DefaultEngine) RestoreStoredPath(ctx context.Context, req RestoreStoredPathRequest) (_ RestoreStoredPathResult, outErr error) { + defer func() { outErr = TranslateError("restore_stored_path", outErr) }() if err := ctx.Err(); err != nil { return RestoreStoredPathResult{}, err } normalized, err := normalizeRestoreStoredPathRequest(req) if err != nil { - return RestoreStoredPathResult{}, err + return RestoreStoredPathResult{}, TranslateErrorAs("restore_stored_path", ErrorInvalidArgument, err) } if err := e.validateRestoreStoredPathDependencies(); err != nil { return RestoreStoredPathResult{}, err diff --git a/internal/engine/snapshot_create.go b/internal/engine/snapshot_create.go index a6df139b..9fa0ee3d 100644 --- a/internal/engine/snapshot_create.go +++ b/internal/engine/snapshot_create.go @@ -16,7 +16,8 @@ type preparedSnapshotCreateRequest struct { snapshotType SnapshotType } -func (e *DefaultEngine) SnapshotCreate(ctx context.Context, req SnapshotCreateRequest) (SnapshotCreateResult, error) { +func (e *DefaultEngine) SnapshotCreate(ctx context.Context, req SnapshotCreateRequest) (_ SnapshotCreateResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_create", outErr) }() if ctx == nil { ctx = context.Background() } @@ -76,10 +77,10 @@ func (e *DefaultEngine) prepareSnapshotCreateRequest(req SnapshotCreateRequest) return preparedSnapshotCreateRequest{}, err } if err := validateSnapshotCreateParent(snapshotID, parentID, len(paths) > 0); err != nil { - return preparedSnapshotCreateRequest{}, err + return preparedSnapshotCreateRequest{}, TranslateErrorAs("snapshot_create", ErrorInvalidArgument, err) } if err := validateSnapshotCreatePaths(paths); err != nil { - return preparedSnapshotCreateRequest{}, err + return preparedSnapshotCreateRequest{}, TranslateErrorAs("snapshot_create", ErrorInvalidArgument, err) } return preparedSnapshotCreateRequest{ diff --git a/internal/engine/snapshot_delete.go b/internal/engine/snapshot_delete.go index f8c4315d..d403d1b0 100644 --- a/internal/engine/snapshot_delete.go +++ b/internal/engine/snapshot_delete.go @@ -13,7 +13,8 @@ type preparedSnapshotDeleteRequest struct { mode SnapshotDeleteMode } -func (e *DefaultEngine) SnapshotDelete(ctx context.Context, req SnapshotDeleteRequest) (SnapshotDeleteResult, error) { +func (e *DefaultEngine) SnapshotDelete(ctx context.Context, req SnapshotDeleteRequest) (_ SnapshotDeleteResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_delete", outErr) }() if ctx == nil { ctx = context.Background() } @@ -23,7 +24,7 @@ func (e *DefaultEngine) SnapshotDelete(ctx context.Context, req SnapshotDeleteRe prepared, err := prepareSnapshotDeleteRequest(req) if err != nil { - return SnapshotDeleteResult{}, err + return SnapshotDeleteResult{}, TranslateErrorAs("snapshot_delete", ErrorInvalidArgument, err) } switch prepared.mode { diff --git a/internal/engine/snapshot_restore.go b/internal/engine/snapshot_restore.go index 510155c8..19235a92 100644 --- a/internal/engine/snapshot_restore.go +++ b/internal/engine/snapshot_restore.go @@ -21,7 +21,8 @@ type preparedSnapshotRestoreRequest struct { restoreSnapshotOpts snapshot.RestoreSnapshotOptions } -func (e *DefaultEngine) SnapshotRestore(ctx context.Context, req SnapshotRestoreRequest) (SnapshotRestoreResult, error) { +func (e *DefaultEngine) SnapshotRestore(ctx context.Context, req SnapshotRestoreRequest) (_ SnapshotRestoreResult, outErr error) { + defer func() { outErr = TranslateError("snapshot_restore", outErr) }() if ctx == nil { ctx = context.Background() } @@ -34,7 +35,7 @@ func (e *DefaultEngine) SnapshotRestore(ctx context.Context, req SnapshotRestore prepared, err := prepareSnapshotRestoreRequest(req, e.config.StoreContext) if err != nil { - return SnapshotRestoreResult{}, err + return SnapshotRestoreResult{}, TranslateErrorAs("snapshot_restore", ErrorInvalidArgument, err) } result, err := snapshot.RestoreSnapshot(ctx, e.config.DB, prepared.snapshotID, prepared.paths, prepared.restoreSnapshotOpts) diff --git a/internal/engine/validation_vs_unsupported_test.go b/internal/engine/validation_vs_unsupported_test.go index a7317f7d..aa05cb73 100644 --- a/internal/engine/validation_vs_unsupported_test.go +++ b/internal/engine/validation_vs_unsupported_test.go @@ -2,7 +2,6 @@ package engine_test import ( "context" - "errors" "testing" "github.com/franchoy/coldkeep/internal/container" @@ -207,32 +206,12 @@ func assertInvalidFileIDFailureSummary(t *testing.T, summary engine.BatchSummary } } -func assertUnsupportedBoundary(t *testing.T, err error, wantMessage string) { - t.Helper() - - if err == nil { - t.Fatal("expected unsupported boundary error") - } - if !errors.Is(err, engine.ErrNotImplemented) { - t.Fatalf("expected ErrNotImplemented-compatible unsupported error, got %v", err) - } - if !engine.IsUnsupported(err) { - t.Fatalf("expected unsupported boundary to classify as unsupported, got %v", err) - } - if err.Error() != wantMessage { - t.Fatalf("expected unsupported message %q, got %q", wantMessage, err.Error()) - } -} - func assertValidationBoundary(t *testing.T, err error, wantMessage string) { t.Helper() if err == nil { t.Fatal("expected validation error") } - if errors.Is(err, engine.ErrNotImplemented) { - t.Fatalf("expected validation error to remain distinct from ErrNotImplemented: %v", err) - } if engine.IsUnsupported(err) { t.Fatalf("expected validation error to remain non-unsupported: %v", err) } diff --git a/internal/observability/read_side_taxonomy_boundary_test.go b/internal/observability/read_side_taxonomy_boundary_test.go index 2a011901..afeebaf1 100644 --- a/internal/observability/read_side_taxonomy_boundary_test.go +++ b/internal/observability/read_side_taxonomy_boundary_test.go @@ -6,14 +6,13 @@ import ( "errors" "testing" - "github.com/franchoy/coldkeep/internal/catalog" "github.com/franchoy/coldkeep/internal/db" "github.com/franchoy/coldkeep/internal/engine" "github.com/franchoy/coldkeep/internal/observability" _ "github.com/mattn/go-sqlite3" ) -func TestInspectDirectObservabilityFailuresRemainOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestInspectDirectObservabilityFailuresRemainOutsideUnsupportedClassification(t *testing.T) { dbconn := openReadSideObservabilityTestDB(t) svc, err := observability.NewService(dbconn) if err != nil { @@ -30,12 +29,9 @@ func TestInspectDirectObservabilityFailuresRemainOutsideUnsupportedAndDeferredCl if engine.IsUnsupported(err) { t.Fatalf("expected missing inspect target to stay outside unsupported classification: %v", err) } - if catalog.IsDeferred(err) { - t.Fatalf("expected missing inspect target to stay outside deferred classification: %v", err) - } } -func TestInspectInvalidTargetRemainsOutsideUnsupportedAndDeferredClassification(t *testing.T) { +func TestInspectInvalidTargetRemainsOutsideUnsupportedClassification(t *testing.T) { dbconn := openReadSideObservabilityTestDB(t) svc, err := observability.NewService(dbconn) if err != nil { @@ -52,12 +48,9 @@ func TestInspectInvalidTargetRemainsOutsideUnsupportedAndDeferredClassification( if engine.IsUnsupported(err) { t.Fatalf("expected invalid target to remain outside unsupported classification: %v", err) } - if catalog.IsDeferred(err) { - t.Fatalf("expected invalid target to remain outside deferred classification: %v", err) - } } -func TestInspectUnsupportedEntityRemainsOutsideEngineUnsupportedAndCatalogDeferred(t *testing.T) { +func TestInspectUnsupportedEntityRemainsOutsideEngineUnsupported(t *testing.T) { dbconn := openReadSideObservabilityTestDB(t) svc, err := observability.NewService(dbconn) if err != nil { @@ -74,9 +67,6 @@ func TestInspectUnsupportedEntityRemainsOutsideEngineUnsupportedAndCatalogDeferr if engine.IsUnsupported(err) { t.Fatalf("expected unsupported inspect entity to stay outside engine unsupported classification: %v", err) } - if catalog.IsDeferred(err) { - t.Fatalf("expected unsupported inspect entity to stay outside catalog deferred classification: %v", err) - } } func openReadSideObservabilityTestDB(t *testing.T) *sql.DB { From 5fd89844e98c8ac2a6c5e4118b5f11810b5c5c68 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 17:01:16 +0200 Subject: [PATCH 21/27] Complete compatibility and adversarial regression --- .github/workflows/ci.yml | 17 +++- CHANGELOG.md | 12 +++ cmd/coldkeep/doctor_engine_routing_test.go | 17 ++++ cmd/coldkeep/stored_path_engine_adapters.go | 2 +- docs/release/v1.13/README.md | 2 +- docs/release/v1.13/v1.13.12-phase-list.md | 6 +- ...ompatibility-and-adversarial-regression.md | 87 +++++++++++++++++++ docs/release/v1.13/v1.13.12-scope.md | 9 +- .../v1.13/v1.13.12-validation-checklist.md | 20 ++++- scripts/audit_ci_enforcement.sh | 12 +++ scripts/audit_ci_enforcement_test.go | 14 +++ 11 files changed, 187 insertions(+), 11 deletions(-) create mode 100644 docs/release/v1.13/v1.13.12-phase20-compatibility-and-adversarial-regression.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bc8f8b8..a0bd9c35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -298,10 +298,15 @@ jobs: "github.com/franchoy/coldkeep/internal/catalog": [ r"^TestCatalogContractFindLogicalFileAcrossBackends/postgres$", r"^TestCatalogContractFindPhysicalFilesAcrossBackends/postgres$", + r"^TestCatalogContractCurrentFileQueriesAcrossBackends/postgres$", + r"^TestCatalogContractRepositoryConfigurationAcrossBackends/postgres$", r"^TestCatalogContractFindSnapshotAcrossBackends/postgres$", r"^TestCatalogContractListSnapshotsAcrossBackends/postgres$", r"^TestCatalogContractLoadReachabilityRootsAcrossBackends/postgres$", - r"^TestCatalogContractDeferredMethodsAcrossBackends/postgres$", + r"^TestCatalogContractSnapshotGraphAcrossBackends/postgres$", + r"^TestCatalogContractGCPlansAcrossBackends/postgres$", + r"^TestCatalogContractChunkPlacementsAcrossBackends/postgres$", + r"^TestCatalogContractRestorePlansAcrossBackends/postgres$", ], "github.com/franchoy/coldkeep/internal/db": [ r"^TestEnsurePostgresSchemaAutoMigratesVersionElevenToTwelve$", @@ -321,6 +326,10 @@ jobs: ], "github.com/franchoy/coldkeep/internal/engine": [ r"^TestRemoveByIDPostgresPreservesSharedChunks$", + r"^TestEngineCurrentFilesAndConfigurationAcrossBackends/postgres$", + r"^TestEngineDoctorAcrossBackends/postgres$", + r"^TestEngineGarbageCollectionPlanAcrossBackends/postgres$", + r"^TestEngineStoreFolderAcrossBackends/postgres$", r"^TestEngineReadStatsAndInspectAcrossBackends/postgres$", r"^TestEngineReadSnapshotViewsAcrossBackends/postgres$", r"^TestEngineReadVerifyAcrossBackends/postgres$", @@ -332,6 +341,12 @@ jobs: r"^TestEngineMutationRestoreAcrossBackends/postgres$", r"^TestEngineMutationErrorsAcrossBackends/postgres$", r"^TestEngineGCDryRunAcrossBackends/postgres$", + r"^TestEngineRecoverAcrossBackendsAndIsIdempotent/postgres$", + r"^TestEngineRecoverFaultAndCancellationAcrossBackends/postgres$", + r"^TestEngineRepairAcrossBackends/postgres$", + r"^TestEngineRepairValidationOrderingAcrossBackends/postgres$", + r"^TestEngineRepairTypedBoundaryErrors/postgres$", + r"^TestEngineRepairFailFastOnInvariantViolation/postgres$", ], "github.com/franchoy/coldkeep/internal/maintenance": [ r"^TestGCAdvisoryLockUsesDedicatedSessionAndReleases$", diff --git a/CHANGELOG.md b/CHANGELOG.md index df9fdfe6..7af81837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,18 @@ project, do not start here; start with [README.md](README.md). engine-owned DTOs, exact tagged dynamic values, ordered trace events, and a complete Verify summary while preserving the existing renderer output through compatibility projection. +- Completed the catalog graph, placement, restore-plan, and GC-plan contracts + on SQLite and PostgreSQL and adopted each in its production path. Completed + engine ownership for folder store, list/search, configuration, snapshots, + inspect/stats/verify, repair, recovery/startup recovery, Doctor, and live GC + planning behind an application composition boundary and enforced thin-CLI + dependency guards. +- Completed the isolated compatibility and adversarial regression phase. Its + full PostgreSQL matrix caught and corrected Doctor session-open exit/message + projection drift; the restarted plain matrix, AES-GCM matrix, G1–G17 + adversarial suite, coordination/advisory-session proofs, legacy fixture, and + post-correction full/race suites pass. Phase 21 exact-head candidate evidence + remains unrun. ------------------------------------------------------------------------ diff --git a/cmd/coldkeep/doctor_engine_routing_test.go b/cmd/coldkeep/doctor_engine_routing_test.go index 12689106..ece72cfc 100644 --- a/cmd/coldkeep/doctor_engine_routing_test.go +++ b/cmd/coldkeep/doctor_engine_routing_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/franchoy/coldkeep/internal/application" "github.com/franchoy/coldkeep/internal/engine" ) @@ -70,6 +71,22 @@ func TestRunDoctorCommandPreservesStageExitClasses(t *testing.T) { } } +func TestRunDoctorCommandClassifiesSessionOpenFailureAsRecovery(t *testing.T) { + original := openApplicationSessionPhase + t.Cleanup(func() { openApplicationSessionPhase = original }) + openApplicationSessionPhase = func(application.Request) (commandSession, error) { + return nil, errors.New("dial failed") + } + + err := runDoctorCommand(parsedCommandLine{method: "doctor", flags: map[string][]string{}}, outputModeJSON) + if err == nil || classifyExitCode(err) != exitRecovery { + t.Fatalf("session-open error=%v exit=%d want %d", err, classifyExitCode(err), exitRecovery) + } + if !strings.Contains(err.Error(), "doctor recovery phase failed: failed to connect to DB: dial failed") { + t.Fatalf("session-open message changed: %v", err) + } +} + func installDoctorCommandStubs(t *testing.T, dbconn *sql.DB, stub stubCommandEngine) { t.Helper() originalConnect := connectDoctorDBPhase diff --git a/cmd/coldkeep/stored_path_engine_adapters.go b/cmd/coldkeep/stored_path_engine_adapters.go index 352812fc..bcbe77fb 100644 --- a/cmd/coldkeep/stored_path_engine_adapters.go +++ b/cmd/coldkeep/stored_path_engine_adapters.go @@ -38,7 +38,7 @@ func recoveryReportFromEngine(result engine.RecoverResult) recovery.Report { func executeDoctorEngine(containersDir, verifyLevel string) (engine.DoctorResult, error) { session, err := openCommandSession("doctor", false, containersDir) if err != nil { - return engine.DoctorResult{}, fmt.Errorf("failed to connect to DB: %w", err) + return engine.DoctorResult{FailedStage: engine.DoctorStageRecovery}, fmt.Errorf("doctor recovery phase failed: failed to connect to DB: %w", err) } defer func() { _ = session.Close() }() ctx, cancel := session.OperationContext(context.Background()) diff --git a/docs/release/v1.13/README.md b/docs/release/v1.13/README.md index 9b631cd3..d9ffa0f8 100644 --- a/docs/release/v1.13/README.md +++ b/docs/release/v1.13/README.md @@ -155,7 +155,7 @@ All v1.13.0 phases stay on `release/v1.13.0` until the full release gate is gree `507859daccf25594142c61e5ab8209a751fb579a`. It must complete the deferred catalog planning APIs, production engine ownership, neutral contracts, and enforceable thin-CLI boundary without changing storage semantics. Phases - 0–19 are complete; Phase 20 is next. + 0–20 are complete; Phase 21 exact-head candidate validation is next. - The updated `v1.13.x-release-train.md` is the authoritative current plan; final v1.x completion remains gated by v1.13.12 and v1.13.13. v1.13.13 is an audit-only final handoff gate, and v2.x implementation is not authorized. diff --git a/docs/release/v1.13/v1.13.12-phase-list.md b/docs/release/v1.13/v1.13.12-phase-list.md index 93c1e398..1e3d8a3a 100644 --- a/docs/release/v1.13/v1.13.12-phase-list.md +++ b/docs/release/v1.13/v1.13.12-phase-list.md @@ -129,11 +129,13 @@ ## Phase 20 — Compatibility and Adversarial Regression -**Status:** Next +**Status:** Complete + +- Evidence: [Phase 20 compatibility and adversarial regression](v1.13.12-phase20-compatibility-and-adversarial-regression.md). ## Phase 21 — Exact-Head Candidate Release Gate -**Status:** Not started +**Status:** Next Phase 21 produces candidate-level PASS/FAIL and merge-authorization readiness only. It cannot pre-claim merge, main, tag, publication, or branch-deletion diff --git a/docs/release/v1.13/v1.13.12-phase20-compatibility-and-adversarial-regression.md b/docs/release/v1.13/v1.13.12-phase20-compatibility-and-adversarial-regression.md new file mode 100644 index 00000000..80a49f85 --- /dev/null +++ b/docs/release/v1.13/v1.13.12-phase20-compatibility-and-adversarial-regression.md @@ -0,0 +1,87 @@ +# Coldkeep v1.13.12 Phase 20 — Compatibility and Adversarial Regression + +**Release:** `v1.13.12 — Engine and Catalog Completion` +**Phase:** `20 — Compatibility and Adversarial Regression` +**Status:** Complete +**Branch:** `release/v1.13.12` + +## Compatibility finding and correction + +The first complete isolated PostgreSQL plain-codec integration attempt found a +real public compatibility regression in the newly routed Doctor startup path: +an application-session open failure returned generic exit code `1` instead of +the frozen recovery exit code `4`. After restoring that classification, the +second complete attempt found the associated message had also lost the frozen +`doctor recovery phase failed:` prefix. + +The adapter now classifies an application-session open failure as Doctor's +recovery stage and preserves the exact established message. Focused JSON +subprocess contracts passed after the correction, followed by a complete fresh +plain-codec integration run. No storage, schema, transaction, repository, +backend-default, or engine behavior changed. + +## CI enforcement closure + +Required CI now selects the active v1.13.12 PostgreSQL catalog contracts for +current files, configuration, snapshot graphs, GC plans, placements, and +restore plans, plus direct engine contracts for current files/configuration, +Doctor, GC planning, folder store, recovery, and repair. The CI-enforcement +audit requires each exact marker, and mutation tests prove removal of both a +catalog and an engine marker fails closed. + +The stale deferred-catalog selector was removed. Static release, CI matrix, +versioned-row-writer, source-format, and diff checks all pass. + +## Regression evidence + +All fresh output was confined to +`/tmp/coldkeep-v11312-phase20.V970Of/`, with separate storage roots and +file-backed SQLite databases per stage. + +- Complete SQLite production-package race matrices: plain and AES-GCM pass. +- Complete PostgreSQL integration matrices: plain and AES-GCM pass. +- Complete long-run G1–G17 adversarial race matrix: pass. +- Exact subprocess contention, killed-holder release, live-GC exclusion, and + deterministic PostgreSQL interleaving selectors: pass in plain and AES-GCM + where applicable, with no skips. +- PostgreSQL dedicated advisory-session acquisition, operation-failure + release, cleanup-failure discard, and single-connection fail-closed tests: + pass. +- Same-process coordinator and Linux native lock lifecycle, contention, + unsafe-artifact, successor-protection, and independent-repository tests: + pass. +- Exact legacy post-migration store/restore/snapshot integration fixture: + pass. +- Linux execution of the hosted path-safety, storage restore, engine restore, + and snapshot restore cross-platform selectors: pass. Native macOS and + Windows runtime proof remains part of the exact-head hosted gate. +- Fresh post-correction `go test -count=1 ./...`: pass. +- Fresh post-correction `go test -race -count=1 ./cmd/... ./internal/...`: + pass. + +The two nonzero full plain attempts were complete-run compatibility findings, +not environmental retries: first the exit class, then the remaining exact +message prefix. Each correction changed tracked candidate code, and the final +plain matrix restarted from an exact freshly created scratch database. + +## Isolation and cleanup + +The review created only these exact PostgreSQL scratch databases: + +- `coldkeep_v11312_p20_plain_v970of` +- `coldkeep_v11312_p20_aes_v970of` +- `coldkeep_v11312_p20_coord_v970of` +- `coldkeep_v11312_p20_legacy_v970of` + +They were created through the `postgres` maintenance database. Cleanup +terminated connections only to those exact names, dropped each exact name, +and a final exact-name query returned no rows. No ordinary database was reset, +truncated, migrated, terminated, or dropped. + +The repository storage fingerprint remains +`8be23869e2f1a165db248ace61b91e8e487573485001af838554f627d4432b92`. +The evidence root is retained for Phase 20 review. + +Phase 21 is authorized to freeze and validate one exact candidate SHA. Phase +20 does not provide candidate-level PASS, merge authorization, publication, +v1.x closure, or v2.x handoff. diff --git a/docs/release/v1.13/v1.13.12-scope.md b/docs/release/v1.13/v1.13.12-scope.md index fb187b15..a9887cd1 100644 --- a/docs/release/v1.13/v1.13.12-scope.md +++ b/docs/release/v1.13/v1.13.12-scope.md @@ -11,10 +11,11 @@ storage semantics or public operator contracts. ## Current phase -Phases 0–19 are Complete. Phase 20 is Next: run the frozen compatibility and -adversarial regression matrix and prove that the architecture completion did -not change public, storage, database, lock, transaction, or failure-order -semantics. +Phases 0–20 are Complete. Phase 21 is Next: freeze one exact candidate SHA and +run the complete isolated local and hosted candidate release gate. Phase 20 +proved the architecture completion preserves public, storage, database, lock, +transaction, and failure-order semantics after correcting one Doctor CLI +compatibility projection at the adapter boundary. ## Required outcomes diff --git a/docs/release/v1.13/v1.13.12-validation-checklist.md b/docs/release/v1.13/v1.13.12-validation-checklist.md index fd39c1d4..37e1e588 100644 --- a/docs/release/v1.13/v1.13.12-validation-checklist.md +++ b/docs/release/v1.13/v1.13.12-validation-checklist.md @@ -314,11 +314,27 @@ ## Phase 20 — Compatibility and Adversarial Regression -**Phase status:** Next +**Phase status:** Complete + +- [x] Frozen CLI, JSON, exit-code, repository-format, legacy/packed layout, + snapshot, repair, recovery, GC, and coordination contracts pass after the + Doctor adapter's established recovery classification and message were + restored. +- [x] Complete plain/AES SQLite production race and PostgreSQL integration + matrices pass in isolated storage/database state. +- [x] Long-run G1–G17 adversarial coverage and the exact subprocess + contention, killed-holder, live-GC, and deterministic interleaving selectors + pass with no skips. +- [x] Same-process/native Linux locks and PostgreSQL advisory-session ownership + retain their lifecycle and fail-closed guarantees. +- [x] Legacy compatibility and all Linux-hosted cross-platform selectors pass; + macOS/Windows native runtime remains an exact-head hosted Phase 21 proof. +- [x] All exact review-created PostgreSQL databases are dropped and the + repository storage fingerprint is unchanged. ## Phase 21 — Exact-Head Candidate Release Gate -**Phase status:** Not started +**Phase status:** Next ## Validation isolation diff --git a/scripts/audit_ci_enforcement.sh b/scripts/audit_ci_enforcement.sh index b56ecf5b..dea4e849 100755 --- a/scripts/audit_ci_enforcement.sh +++ b/scripts/audit_ci_enforcement.sh @@ -787,6 +787,18 @@ check_local_workflow() { require_content_pattern "$postgres_internal_contracts_block" 'expected PostgreSQL pass missing' 'PostgreSQL internal package contracts require PostgreSQL test pass events' || check_status=1 require_content_pattern "$postgres_internal_contracts_block" 'TestSCH001AndSCH002BootstrapVersionAndIdempotency/postgres' 'PostgreSQL internal package contracts prove Phase 5 bootstrap execution' || check_status=1 require_content_pattern "$postgres_internal_contracts_block" 'TestSCH009PostgresVersionElevenAutoMigration/postgres' 'PostgreSQL internal package contracts prove Phase 5 migration execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestCatalogContractCurrentFileQueriesAcrossBackends/postgres' 'PostgreSQL internal package contracts prove catalog current-file execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestCatalogContractRepositoryConfigurationAcrossBackends/postgres' 'PostgreSQL internal package contracts prove catalog configuration execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestCatalogContractSnapshotGraphAcrossBackends/postgres' 'PostgreSQL internal package contracts prove catalog graph execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestCatalogContractChunkPlacementsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove catalog placement execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestCatalogContractRestorePlansAcrossBackends/postgres' 'PostgreSQL internal package contracts prove catalog restore-plan execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestCatalogContractGCPlansAcrossBackends/postgres' 'PostgreSQL internal package contracts prove catalog GC-plan execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineCurrentFilesAndConfigurationAcrossBackends/postgres' 'PostgreSQL internal package contracts prove engine current-file and configuration execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineStoreFolderAcrossBackends/postgres' 'PostgreSQL internal package contracts prove engine folder-store execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineGarbageCollectionPlanAcrossBackends/postgres' 'PostgreSQL internal package contracts prove engine GC-plan execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineRepairAcrossBackends/postgres' 'PostgreSQL internal package contracts prove engine repair execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineRecoverAcrossBackendsAndIsIdempotent/postgres' 'PostgreSQL internal package contracts prove engine recovery execution' || check_status=1 + require_content_pattern "$postgres_internal_contracts_block" 'TestEngineDoctorAcrossBackends/postgres' 'PostgreSQL internal package contracts prove engine Doctor execution' || check_status=1 require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadStatsAndInspectAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 stats and inspect execution' || check_status=1 require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadSnapshotViewsAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 snapshot-view execution' || check_status=1 require_content_pattern "$postgres_internal_contracts_block" 'TestEngineReadVerifyAcrossBackends/postgres' 'PostgreSQL internal package contracts prove Phase 7 verification execution' || check_status=1 diff --git a/scripts/audit_ci_enforcement_test.go b/scripts/audit_ci_enforcement_test.go index 7c16e78e..418fb0cb 100644 --- a/scripts/audit_ci_enforcement_test.go +++ b/scripts/audit_ci_enforcement_test.go @@ -820,6 +820,20 @@ func TestAuditCIEnforcementRejectsPhase18RequiredProofMutations(t *testing.T) { replacement: "TestMutationRowsAffectedContractAcrossBackends/sqlite", wantMessage: "PostgreSQL internal package contracts prove Phase 17 mutation-cardinality execution", }, + { + name: "v1.13.12 PostgreSQL catalog planning marker", + anchor: " - name: Run required PostgreSQL internal package contracts\n", + old: "TestCatalogContractRestorePlansAcrossBackends/postgres", + replacement: "TestCatalogContractRestorePlansAcrossBackends/sqlite", + wantMessage: "PostgreSQL internal package contracts prove catalog restore-plan execution", + }, + { + name: "v1.13.12 PostgreSQL engine Doctor marker", + anchor: " - name: Run required PostgreSQL internal package contracts\n", + old: "TestEngineDoctorAcrossBackends/postgres", + replacement: "TestEngineDoctorAcrossBackends/sqlite", + wantMessage: "PostgreSQL internal package contracts prove engine Doctor execution", + }, { name: "Unix native contention source", old: "func TestNativeLockContentionAndReacquire", From 39bed341165d2ef45c09a94f092bc7f0e5465cd0 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 18:52:05 +0200 Subject: [PATCH 22/27] fix(ci): reconcile v1.13.12 hosted quality --- .github/workflows/ci.yml | 3 + cmd/coldkeep/legacy_application_seams_test.go | 28 +--- cmd/coldkeep/legacy_doctor_seams_test.go | 3 - .../legacy_observability_verify_seams_test.go | 2 - cmd/coldkeep/legacy_repair_seams_test.go | 2 - internal/recovery/system_recovery.go | 18 --- scripts/audit_ci_enforcement.sh | 14 ++ scripts/audit_ci_enforcement_test.go | 122 ++++++++++++++++++ scripts/run_candidate_lint_gate.sh | 117 +++++++++++++++++ 9 files changed, 260 insertions(+), 49 deletions(-) create mode 100755 scripts/run_candidate_lint_gate.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0bd9c35..73baaea3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,9 @@ jobs: - name: Enforce versioned row writer scope run: bash scripts/check_versioned_row_writers.sh + + - name: Test candidate lint gate contract + run: go test -count=1 ./scripts -run '^TestCandidateLintGate' - name: Lint uses: golangci/golangci-lint-action@v9 diff --git a/cmd/coldkeep/legacy_application_seams_test.go b/cmd/coldkeep/legacy_application_seams_test.go index 70272e25..2f161ced 100644 --- a/cmd/coldkeep/legacy_application_seams_test.go +++ b/cmd/coldkeep/legacy_application_seams_test.go @@ -132,11 +132,12 @@ func openLegacyApplicationSession(req application.Request) (commandSession, erro func openLegacyDBSession(req application.Request) (commandSession, error) { connector := connectListSearchDBPhase - if req.Operation == "repair" { + switch req.Operation { + case "repair": connector = connectRepairDBPhase - } else if req.Operation == "recovery" { + case "recovery": connector = connectRecoveryDBPhase - } else if req.Operation == "doctor" { + case "doctor": connector = connectDoctorDBPhase } dbconn, err := connector() @@ -299,24 +300,3 @@ func (e legacyPerItemEngine) Restore(_ context.Context, req engine.RestoreReques } return result, nil } - -func executeRestoreDryRunItem(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool) batch.ItemResult { - result, err := restoreByIDPhase(sgctx, fileID, outputDir, overwrite, true) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: fmt.Sprintf("file ID %d not found", fileID)} - } - return batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} - } - return batch.ItemResult{ID: fileID, Status: batch.ResultPlanned, Message: fmt.Sprintf("would restore -> %s", result.OutputPath), OriginalName: result.OriginalName, OutputPath: result.OutputPath} -} - -func executeRestoreItem(sgctx *storage.StorageContext, fileID int64, outputDir string, overwrite bool) batch.ItemResult { - result, err := restoreByIDPhase(sgctx, fileID, outputDir, overwrite, false) - if err != nil { - item := batch.ItemResult{ID: fileID, Status: batch.ResultFailed, Message: err.Error()} - annotateBatchFailureFromError(err, &item) - return item - } - return batch.ItemResult{ID: fileID, Status: batch.ResultSuccess, Message: "restored", OriginalName: result.OriginalName, OutputPath: result.OutputPath} -} diff --git a/cmd/coldkeep/legacy_doctor_seams_test.go b/cmd/coldkeep/legacy_doctor_seams_test.go index abb58f6b..6299a73e 100644 --- a/cmd/coldkeep/legacy_doctor_seams_test.go +++ b/cmd/coldkeep/legacy_doctor_seams_test.go @@ -18,9 +18,6 @@ var doctorSchemaVersionPhase = db.QueryCurrentSchemaVersion var doctorVerifyPhase = maintenance.VerifyCommandWithContainersDir var doctorSystemAuditPhase = maintenance.CollectSystemAuditSummary -var productionDoctorDBConnector = connectDoctorDBPhase -var productionDoctorCommandEngine = newDoctorCommandEngine - func init() { connectDoctorDBPhase = func() (*sql.DB, error) { return sql.Open("sqlite3", ":memory:") } newDoctorCommandEngine = func(*sql.DB, string) (engine.Engine, error) { diff --git a/cmd/coldkeep/legacy_observability_verify_seams_test.go b/cmd/coldkeep/legacy_observability_verify_seams_test.go index 525b9929..8ff95b8a 100644 --- a/cmd/coldkeep/legacy_observability_verify_seams_test.go +++ b/cmd/coldkeep/legacy_observability_verify_seams_test.go @@ -31,8 +31,6 @@ var verifySummaryPhase = func(_ *sql.DB, _ string, _ int64) (verifyOutputSummary return verifyOutputSummary{}, nil } -var productionVerifyCommandEngine = newVerifyCommandEngine - func init() { newVerifyCommandEngine = func(sgctx storage.StorageContext) (engine.Engine, error) { return legacyVerifyTestEngine{db: sgctx.DB}, nil diff --git a/cmd/coldkeep/legacy_repair_seams_test.go b/cmd/coldkeep/legacy_repair_seams_test.go index 9aebc12b..e3243938 100644 --- a/cmd/coldkeep/legacy_repair_seams_test.go +++ b/cmd/coldkeep/legacy_repair_seams_test.go @@ -13,8 +13,6 @@ import ( var repairLogicalRefCountsPhase = maintenance.RepairLogicalRefCountsResultRun var repairChunkLiveRefCountsPhase = maintenance.RepairChunkLiveRefCountsResultRun -var productionRepairCommandEngine = newRepairCommandEngine -var productionRepairDBConnector = connectRepairDBPhase func init() { connectRepairDBPhase = func() (*sql.DB, error) { return sql.Open("sqlite3", ":memory:") } diff --git a/internal/recovery/system_recovery.go b/internal/recovery/system_recovery.go index e2f9ce61..b59bc243 100644 --- a/internal/recovery/system_recovery.go +++ b/internal/recovery/system_recovery.go @@ -196,22 +196,10 @@ func abortProcessingChunksWithContext(ctx context.Context, dbconn *sql.DB, stats return nil } -func recoverSealingContainers(dbconn *sql.DB, containersDir string, stats *recoveryStats) error { - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() - return recoverSealingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) -} - func recoverSealingContainersWithContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats) error { return recoverSealingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) } -func recoverSealingContainersWithFS(dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() - return recoverSealingContainersWithFSContext(ctx, dbconn, containersDir, stats, fsys) -} - func recoverSealingContainersWithFSContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats, fsys fsx.FS) error { logRecoveryEvent("recover_sealing_containers_start") @@ -394,12 +382,6 @@ func quarantineMissingContainersWithFSContext(ctx context.Context, dbconn *sql.D return nil } -func quarantineCorruptActiveContainerTails(dbconn *sql.DB, containersDir string, stats *recoveryStats) error { - ctx, cancel := db.NewOperationContext(context.Background()) - defer cancel() - return quarantineCorruptActiveContainerTailsWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) -} - func quarantineCorruptActiveContainerTailsWithContext(ctx context.Context, dbconn *sql.DB, containersDir string, stats *recoveryStats) error { return quarantineCorruptActiveContainerTailsWithFSContext(ctx, dbconn, containersDir, stats, fsx.Default()) } diff --git a/scripts/audit_ci_enforcement.sh b/scripts/audit_ci_enforcement.sh index dea4e849..79eb9d05 100755 --- a/scripts/audit_ci_enforcement.sh +++ b/scripts/audit_ci_enforcement.sh @@ -74,6 +74,7 @@ CODEQL_WORKFLOW_FILE="${COLDKEEP_CODEQL_WORKFLOW_FILE:-$REPO_ROOT/.github/workfl BENCHMARK_BASELINE_WORKFLOW_FILE="${COLDKEEP_BENCHMARK_BASELINE_WORKFLOW_FILE:-$REPO_ROOT/.github/workflows/benchmark-baseline.yml}" BENCHMARK_GATE_FILE="${COLDKEEP_BENCHMARK_GATE_FILE:-$REPO_ROOT/scripts/benchmark_gate.py}" TIMING_VALIDATOR_FILE="${COLDKEEP_TIMING_VALIDATOR_FILE:-$REPO_ROOT/scripts/validate_regression_thresholds.py}" +CANDIDATE_LINT_GATE_FILE="${COLDKEEP_CANDIDATE_LINT_GATE_FILE:-$REPO_ROOT/scripts/run_candidate_lint_gate.sh}" VALIDATION_MATRIX_FILE="${COLDKEEP_VALIDATION_MATRIX_FILE:-$REPO_ROOT/VALIDATION_MATRIX.md}" PAIRED_REFERENCE_MANIFEST_FILE="${COLDKEEP_PAIRED_REFERENCE_MANIFEST_FILE:-$REPO_ROOT/benchmarks/paired/reference-v1.13.json}" PAIRED_THRESHOLD_POLICY_FILE="${COLDKEEP_PAIRED_THRESHOLD_POLICY_FILE:-$REPO_ROOT/benchmarks/paired/threshold-policy-v1.13.json}" @@ -449,6 +450,19 @@ check_local_workflow() { echo "[audit] checking local workflow invariants" require_pattern "$WORKFLOW_FILE" 'name: CI' 'CI workflow file' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'uses:\s*golangci/golangci-lint-action@v9' 'hosted quality uses golangci-lint action v9' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'version:\s*v2\.6\.2' 'hosted quality pins golangci-lint v2.6.2' || check_status=1 + require_pattern "$WORKFLOW_FILE" 'name:\s*Test candidate lint gate contract' 'hosted quality tests the candidate lint gate contract' || check_status=1 + require_pattern "$WORKFLOW_FILE" "go test -count=1 ./scripts -run '\^TestCandidateLintGate'" 'hosted quality runs candidate lint gate regression tests' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" '^set -Eeuo pipefail$' 'candidate lint gate enables fail-closed shell semantics' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" '^set -o pipefail$' 'candidate lint gate preserves pipeline failures' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" '^readonly EXPECTED_GOLANGCI_LINT_VERSION="2\.6\.2"$' 'candidate lint gate pins golangci-lint v2.6.2' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" 'config path' 'candidate lint gate resolves the effective repository config' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" 'config verify' 'candidate lint gate verifies the effective repository config' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" 'pipeline_status=\("\$\{PIPESTATUS\[@\]\}"\)' 'candidate lint gate captures lint and tee pipeline statuses' || check_status=1 + # shellcheck disable=SC2016 # The audit pattern must match the literal gate variables. + require_pattern "$CANDIDATE_LINT_GATE_FILE" 'grep -Eq -- "\$FINDING_PATTERN" "\$LINT_LOG"' 'candidate lint gate rejects findings independently of wrapper status' || check_status=1 + require_pattern "$CANDIDATE_LINT_GATE_FILE" 'LOCAL_CANDIDATE_LINT=PASS' 'candidate lint gate emits PASS only after evidence verification' || check_status=1 require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^name: Benchmark Gate Calibration and Baseline Capture$' 'manual benchmark calibration workflow' || check_status=1 require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^ workflow_dispatch:$' 'benchmark calibration is manually dispatched' || check_status=1 require_pattern "$BENCHMARK_BASELINE_WORKFLOW_FILE" '^ contents: read$' 'benchmark calibration has read-only repository permission' || check_status=1 diff --git a/scripts/audit_ci_enforcement_test.go b/scripts/audit_ci_enforcement_test.go index 418fb0cb..f78b5430 100644 --- a/scripts/audit_ci_enforcement_test.go +++ b/scripts/audit_ci_enforcement_test.go @@ -27,6 +27,90 @@ func TestAuditCIEnforcementLocalWorkflowRequiresCrossPlatformInNeeds(t *testing. } } +func TestAuditCIEnforcementRequiresCandidateLintParity(t *testing.T) { + workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) + codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) + + t.Run("hosted version pin", func(t *testing.T) { + mutated := strings.Replace(workflow, "version: v2.6.2", "version: v2.6.3", 1) + if mutated == workflow { + t.Fatal("hosted linter version fixture not found") + } + stderr := runAuditLocalOnly(t, mutated, codeqlWorkflow, true) + if !strings.Contains(stderr, "hosted quality pins golangci-lint v2.6.2") { + t.Fatalf("expected hosted linter pin failure, got:\n%s", stderr) + } + }) + + t.Run("pipeline status capture", func(t *testing.T) { + gate := readRepoFile(t, filepath.Join("scripts", "run_candidate_lint_gate.sh")) + mutated := strings.Replace(gate, "pipeline_status=(\"${PIPESTATUS[@]}\")", "pipeline_status=(0 0)", 1) + if mutated == gate { + t.Fatal("candidate lint pipeline fixture not found") + } + gatePath := filepath.Join(t.TempDir(), "run_candidate_lint_gate.sh") + if err := os.WriteFile(gatePath, []byte(mutated), 0o700); err != nil { + t.Fatalf("write candidate lint gate fixture: %v", err) + } + t.Setenv("COLDKEEP_CANDIDATE_LINT_GATE_FILE", gatePath) + stderr := runAuditLocalOnly(t, workflow, codeqlWorkflow, true) + if !strings.Contains(stderr, "candidate lint gate captures lint and tee pipeline statuses") { + t.Fatalf("expected pipeline status audit failure, got:\n%s", stderr) + } + }) +} + +func TestCandidateLintGateRejectsFindingsWhenLinterReturnsSuccess(t *testing.T) { + fakeLinter := writeFakeCandidateLinter(t) + evidenceDir := t.TempDir() + cmd := exec.Command("bash", "scripts/run_candidate_lint_gate.sh", "run", evidenceDir) + cmd.Dir = repoRoot(t) + cmd.Env = append(os.Environ(), + "COLDKEEP_GOLANGCI_LINT_BIN="+fakeLinter, + "FAKE_LINT_OUTPUT=cmd/example.go:7:3: synthetic actionable finding (unused)", + "FAKE_LINT_EXIT=0", + ) + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("expected actionable finding to fail closed, got success:\n%s", output) + } + if strings.Contains(string(output), "LOCAL_CANDIDATE_LINT=PASS") { + t.Fatalf("finding-bearing run emitted PASS:\n%s", output) + } + status, readErr := os.ReadFile(filepath.Join(evidenceDir, "golangci-lint.status")) + if readErr != nil { + t.Fatalf("read lint status: %v", readErr) + } + if string(status) != "FAIL\n" { + t.Fatalf("finding-bearing run status = %q, want FAIL", status) + } + + verify := exec.Command("bash", "scripts/run_candidate_lint_gate.sh", "verify", evidenceDir) + verify.Dir = repoRoot(t) + if verifyOutput, verifyErr := verify.CombinedOutput(); verifyErr == nil { + t.Fatalf("expected finding-bearing evidence verification to fail, got:\n%s", verifyOutput) + } +} + +func TestCandidateLintGateAcceptsCleanPinnedLinter(t *testing.T) { + fakeLinter := writeFakeCandidateLinter(t) + evidenceDir := t.TempDir() + cmd := exec.Command("bash", "scripts/run_candidate_lint_gate.sh", "run", evidenceDir) + cmd.Dir = repoRoot(t) + cmd.Env = append(os.Environ(), + "COLDKEEP_GOLANGCI_LINT_BIN="+fakeLinter, + "FAKE_LINT_OUTPUT=", + "FAKE_LINT_EXIT=0", + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("expected clean pinned linter to pass, got err=%v output:\n%s", err, output) + } + if !strings.Contains(string(output), "LOCAL_CANDIDATE_LINT=PASS") { + t.Fatalf("clean run did not emit PASS:\n%s", output) + } +} + func TestAuditCIEnforcementLocalWorkflowRequiresNativeCoordinationRuntime(t *testing.T) { workflow := readRepoFile(t, filepath.Join(".github", "workflows", "ci.yml")) codeqlWorkflow := readRepoFile(t, filepath.Join(".github", "workflows", "codeql.yml")) @@ -1353,6 +1437,44 @@ func readRepoFile(t *testing.T, relPath string) string { return string(content) } +func writeFakeCandidateLinter(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "golangci-lint") + content := `#!/usr/bin/env bash +set -euo pipefail +case "${1:-}" in + version) + echo "golangci-lint has version 2.6.2 built with test" + ;; + config) + case "${2:-}" in + path) + echo ".golangci.yml" + ;; + verify) + ;; + *) + exit 2 + ;; + esac + ;; + run) + if [[ -n "${FAKE_LINT_OUTPUT:-}" ]]; then + printf '%s\n' "${FAKE_LINT_OUTPUT}" + fi + exit "${FAKE_LINT_EXIT:-0}" + ;; + *) + exit 2 + ;; +esac +` + if err := os.WriteFile(path, []byte(content), 0o700); err != nil { + t.Fatalf("write fake candidate linter: %v", err) + } + return path +} + func repoRoot(t *testing.T) string { t.Helper() diff --git a/scripts/run_candidate_lint_gate.sh b/scripts/run_candidate_lint_gate.sh new file mode 100755 index 00000000..2cb295ef --- /dev/null +++ b/scripts/run_candidate_lint_gate.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set -o pipefail + +readonly EXPECTED_GOLANGCI_LINT_VERSION="2.6.2" + +usage() { + cat <<'EOF' +Usage: scripts/run_candidate_lint_gate.sh EVIDENCE_DIR + +Runs or verifies the fail-closed Phase 21 Go lint stage. EVIDENCE_DIR must be +an existing absolute directory below /tmp that is dedicated to the current +exact-head candidate validation. +EOF +} + +if [[ $# -ne 2 ]]; then + usage >&2 + exit 2 +fi + +readonly MODE="$1" +readonly EVIDENCE_DIR="$2" +if [[ "$MODE" != "run" && "$MODE" != "verify" ]]; then + usage >&2 + exit 2 +fi +if [[ "$EVIDENCE_DIR" != /tmp/* || ! -d "$EVIDENCE_DIR" ]]; then + echo "[candidate-lint] ERROR: EVIDENCE_DIR must be an existing directory below /tmp" >&2 + exit 2 +fi + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd) +readonly SCRIPT_DIR REPO_ROOT +readonly LINT_LOG="$EVIDENCE_DIR/golangci-lint.log" +readonly LINT_STATUS="$EVIDENCE_DIR/golangci-lint.status" +readonly LINT_VERSION="$EVIDENCE_DIR/golangci-lint.version" +readonly LINTER_BIN="${COLDKEEP_GOLANGCI_LINT_BIN:-golangci-lint}" +readonly FINDING_PATTERN='^[^[:space:]].*:[0-9]+:[0-9]+: .+ \([[:alnum:]_-]+\)$|^[1-9][0-9]* issues?:$' + +verify_evidence() { + if [[ ! -f "$LINT_STATUS" || "$(<"$LINT_STATUS")" != "PASS" ]]; then + echo "[candidate-lint] ERROR: lint status is not PASS" >&2 + return 1 + fi + if [[ ! -f "$LINT_VERSION" || "$(<"$LINT_VERSION")" != "$EXPECTED_GOLANGCI_LINT_VERSION" ]]; then + echo "[candidate-lint] ERROR: lint version evidence is not $EXPECTED_GOLANGCI_LINT_VERSION" >&2 + return 1 + fi + if [[ ! -f "$LINT_LOG" ]]; then + echo "[candidate-lint] ERROR: lint log is missing" >&2 + return 1 + fi + if grep -Eq -- "$FINDING_PATTERN" "$LINT_LOG"; then + echo "[candidate-lint] ERROR: lint log contains actionable findings" >&2 + return 1 + fi + echo "LOCAL_CANDIDATE_LINT=PASS" +} + +if [[ "$MODE" == "verify" ]]; then + verify_evidence + exit +fi + +umask 077 +printf 'FAIL\n' > "$LINT_STATUS" +: > "$LINT_LOG" +: > "$LINT_VERSION" + +cd "$REPO_ROOT" +version_output="$($LINTER_BIN version 2>&1)" || { + printf '%s\n' "$version_output" >&2 + echo "[candidate-lint] ERROR: cannot execute golangci-lint" >&2 + exit 1 +} +if [[ ! "$version_output" =~ (^|[[:space:]])version[[:space:]]+2[.]6[.]2([[:space:]]|$) ]]; then + printf '%s\n' "$version_output" >&2 + echo "[candidate-lint] ERROR: expected golangci-lint $EXPECTED_GOLANGCI_LINT_VERSION" >&2 + exit 1 +fi +printf '%s\n' "$EXPECTED_GOLANGCI_LINT_VERSION" > "$LINT_VERSION" + +config_path="$($LINTER_BIN config path 2>&1)" || { + printf '%s\n' "$config_path" >&2 + echo "[candidate-lint] ERROR: cannot resolve golangci-lint config" >&2 + exit 1 +} +if [[ "$(realpath -e -- "$config_path")" != "$REPO_ROOT/.golangci.yml" ]]; then + echo "[candidate-lint] ERROR: effective config is not $REPO_ROOT/.golangci.yml" >&2 + exit 1 +fi +"$LINTER_BIN" config verify + +set +e +NO_COLOR=1 "$LINTER_BIN" run 2>&1 | tee "$LINT_LOG" +pipeline_status=("${PIPESTATUS[@]}") +set -e +lint_status="${pipeline_status[0]}" +tee_status="${pipeline_status[1]}" + +if [[ "$tee_status" -ne 0 ]]; then + echo "[candidate-lint] ERROR: failed to write lint evidence" >&2 + exit 1 +fi +if grep -Eq -- "$FINDING_PATTERN" "$LINT_LOG"; then + echo "[candidate-lint] ERROR: golangci-lint reported actionable findings" >&2 + exit 1 +fi +if [[ "$lint_status" -ne 0 ]]; then + echo "[candidate-lint] ERROR: golangci-lint exited with status $lint_status" >&2 + exit "$lint_status" +fi + +printf 'PASS\n' > "$LINT_STATUS" +verify_evidence From 771b28fff34fbdd159a82c90edbe46bec7d73a85 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 19:44:55 +0200 Subject: [PATCH 23/27] fix(ci): quote candidate linter executable --- scripts/audit_ci_enforcement_test.go | 23 ++++++++++++++++++++++- scripts/run_candidate_lint_gate.sh | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/audit_ci_enforcement_test.go b/scripts/audit_ci_enforcement_test.go index f78b5430..701761f3 100644 --- a/scripts/audit_ci_enforcement_test.go +++ b/scripts/audit_ci_enforcement_test.go @@ -94,11 +94,16 @@ func TestCandidateLintGateRejectsFindingsWhenLinterReturnsSuccess(t *testing.T) func TestCandidateLintGateAcceptsCleanPinnedLinter(t *testing.T) { fakeLinter := writeFakeCandidateLinter(t) + if !strings.Contains(fakeLinter, " ") { + t.Fatalf("fake linter path %q does not exercise whitespace handling", fakeLinter) + } evidenceDir := t.TempDir() + callLog := filepath.Join(t.TempDir(), "calls.log") cmd := exec.Command("bash", "scripts/run_candidate_lint_gate.sh", "run", evidenceDir) cmd.Dir = repoRoot(t) cmd.Env = append(os.Environ(), "COLDKEEP_GOLANGCI_LINT_BIN="+fakeLinter, + "FAKE_LINT_CALL_LOG="+callLog, "FAKE_LINT_OUTPUT=", "FAKE_LINT_EXIT=0", ) @@ -109,6 +114,15 @@ func TestCandidateLintGateAcceptsCleanPinnedLinter(t *testing.T) { if !strings.Contains(string(output), "LOCAL_CANDIDATE_LINT=PASS") { t.Fatalf("clean run did not emit PASS:\n%s", output) } + calls, err := os.ReadFile(callLog) + if err != nil { + t.Fatalf("read fake linter call log: %v", err) + } + for _, invocation := range []string{"version\n", "config path\n"} { + if !strings.Contains(string(calls), invocation) { + t.Fatalf("whitespace-path fake linter did not receive %q; calls:\n%s", strings.TrimSpace(invocation), calls) + } + } } func TestAuditCIEnforcementLocalWorkflowRequiresNativeCoordinationRuntime(t *testing.T) { @@ -1439,9 +1453,16 @@ func readRepoFile(t *testing.T, relPath string) string { func writeFakeCandidateLinter(t *testing.T) string { t.Helper() - path := filepath.Join(t.TempDir(), "golangci-lint") + toolDir := filepath.Join(t.TempDir(), "candidate lint tools") + if err := os.MkdirAll(toolDir, 0o700); err != nil { + t.Fatalf("create fake candidate linter directory: %v", err) + } + path := filepath.Join(toolDir, "golangci-lint") content := `#!/usr/bin/env bash set -euo pipefail +if [[ -n "${FAKE_LINT_CALL_LOG:-}" ]]; then + printf '%s\n' "$*" >> "${FAKE_LINT_CALL_LOG}" +fi case "${1:-}" in version) echo "golangci-lint has version 2.6.2 built with test" diff --git a/scripts/run_candidate_lint_gate.sh b/scripts/run_candidate_lint_gate.sh index 2cb295ef..a8e89f8f 100755 --- a/scripts/run_candidate_lint_gate.sh +++ b/scripts/run_candidate_lint_gate.sh @@ -70,7 +70,7 @@ printf 'FAIL\n' > "$LINT_STATUS" : > "$LINT_VERSION" cd "$REPO_ROOT" -version_output="$($LINTER_BIN version 2>&1)" || { +version_output="$("$LINTER_BIN" version 2>&1)" || { printf '%s\n' "$version_output" >&2 echo "[candidate-lint] ERROR: cannot execute golangci-lint" >&2 exit 1 @@ -82,7 +82,7 @@ if [[ ! "$version_output" =~ (^|[[:space:]])version[[:space:]]+2[.]6[.]2([[:spac fi printf '%s\n' "$EXPECTED_GOLANGCI_LINT_VERSION" > "$LINT_VERSION" -config_path="$($LINTER_BIN config path 2>&1)" || { +config_path="$("$LINTER_BIN" config path 2>&1)" || { printf '%s\n' "$config_path" >&2 echo "[candidate-lint] ERROR: cannot resolve golangci-lint config" >&2 exit 1 From bd006171918be4df5fa3dce50901446d118fd748 Mon Sep 17 00:00:00 2001 From: franchoy Date: Wed, 19 Aug 2026 20:26:54 +0200 Subject: [PATCH 24/27] fix(security): bound integer conversions --- internal/catalog/current_files.go | 15 ++++---- internal/catalog/current_files_test.go | 50 ++++++++++++++++++++++++++ internal/cli/render/inspect.go | 6 +++- internal/cli/render/inspect_test.go | 43 ++++++++++++++++++++++ 4 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 internal/catalog/current_files_test.go diff --git a/internal/catalog/current_files.go b/internal/catalog/current_files.go index 6a373599..bd96ff49 100644 --- a/internal/catalog/current_files.go +++ b/internal/catalog/current_files.go @@ -116,15 +116,16 @@ WHERE lf.status = $1`) } func paginateCurrentFiles(refs []CurrentFileRef, page CurrentFilePage) []CurrentFileRef { - start := 0 - if page.Offset != nil && *page.Offset < int64(len(refs)) { - start = int(*page.Offset) + refsLength := int64(len(refs)) + start := int64(0) + if page.Offset != nil && *page.Offset < refsLength { + start = *page.Offset } else if page.Offset != nil { - start = len(refs) + start = refsLength } - end := len(refs) - if page.Limit != nil && *page.Limit < int64(end-start) { - end = start + int(*page.Limit) + end := refsLength + if page.Limit != nil && *page.Limit < end-start { + end = start + *page.Limit } return refs[start:end] } diff --git a/internal/catalog/current_files_test.go b/internal/catalog/current_files_test.go new file mode 100644 index 00000000..bd62b9d1 --- /dev/null +++ b/internal/catalog/current_files_test.go @@ -0,0 +1,50 @@ +package catalog + +import ( + "math" + "reflect" + "testing" +) + +func TestPaginateCurrentFilesBoundaries(t *testing.T) { + refs := []CurrentFileRef{ + {LogicalFileID: 1}, + {LogicalFileID: 2}, + {LogicalFileID: 3}, + } + value := func(n int64) *int64 { return &n } + tests := []struct { + name string + page CurrentFilePage + want []int64 + }{ + {name: "nil offset and limit", want: []int64{1, 2, 3}}, + {name: "zero offset", page: CurrentFilePage{Offset: value(0)}, want: []int64{1, 2, 3}}, + {name: "offset below length", page: CurrentFilePage{Offset: value(1)}, want: []int64{2, 3}}, + {name: "offset equal length", page: CurrentFilePage{Offset: value(3)}, want: []int64{}}, + {name: "offset greater than length", page: CurrentFilePage{Offset: value(4)}, want: []int64{}}, + {name: "maximum int64 offset", page: CurrentFilePage{Offset: value(math.MaxInt64)}, want: []int64{}}, + {name: "zero limit", page: CurrentFilePage{Limit: value(0)}, want: []int64{}}, + {name: "limit below remaining", page: CurrentFilePage{Limit: value(2)}, want: []int64{1, 2}}, + {name: "limit equal remaining", page: CurrentFilePage{Offset: value(1), Limit: value(2)}, want: []int64{2, 3}}, + {name: "limit greater than remaining", page: CurrentFilePage{Offset: value(1), Limit: value(3)}, want: []int64{2, 3}}, + {name: "maximum allowed page size", page: CurrentFilePage{Limit: value(MaxCurrentFilePageSize)}, want: []int64{1, 2, 3}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotRefs := paginateCurrentFiles(refs, tc.page) + got := make([]int64, len(gotRefs)) + for i, ref := range gotRefs { + got[i] = ref.LogicalFileID + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("paginateCurrentFiles() = %v, want %v", got, tc.want) + } + }) + } + + if err := validateCurrentFilePage(CurrentFilePage{Limit: value(MaxCurrentFilePageSize)}); err != nil { + t.Fatalf("MaxCurrentFilePageSize must remain valid: %v", err) + } +} diff --git a/internal/cli/render/inspect.go b/internal/cli/render/inspect.go index c981c2ad..d4f60305 100644 --- a/internal/cli/render/inspect.go +++ b/internal/cli/render/inspect.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "math" "sort" "strconv" "strings" @@ -269,6 +270,9 @@ func toInt64(v any) (int64, bool) { case int64: return n, true case uint: + if uint64(n) > uint64(math.MaxInt64) { + return 0, false + } return int64(n), true case uint8: return int64(n), true @@ -277,7 +281,7 @@ func toInt64(v any) (int64, bool) { case uint32: return int64(n), true case uint64: - if n > uint64(^uint64(0)>>1) { + if n > uint64(math.MaxInt64) { return 0, false } return int64(n), true diff --git a/internal/cli/render/inspect_test.go b/internal/cli/render/inspect_test.go index 152964d4..89f70322 100644 --- a/internal/cli/render/inspect_test.go +++ b/internal/cli/render/inspect_test.go @@ -3,6 +3,8 @@ package render import ( "bytes" "encoding/json" + "math" + "strconv" "strings" "testing" @@ -139,3 +141,44 @@ func TestRenderInspectJSONIsDeterministic(t *testing.T) { t.Fatalf("expected sorted relations by target_id, got %v", got) } } + +func TestToInt64UnsignedBoundaries(t *testing.T) { + tests := []struct { + name string + value any + want int64 + wantOK bool + }{ + {name: "normal uint", value: uint(42), want: 42, wantOK: true}, + {name: "maximum int64 as uint64", value: uint64(math.MaxInt64), want: math.MaxInt64, wantOK: true}, + {name: "one above maximum int64 as uint64", value: uint64(math.MaxInt64) + 1, wantOK: false}, + {name: "maximum uint64", value: uint64(math.MaxUint64), wantOK: false}, + } + if strconv.IntSize == 64 { + tests = append(tests, struct { + name string + value any + want int64 + wantOK bool + }{name: "one above maximum int64 as uint", value: uint(uint64(math.MaxInt64) + 1), wantOK: false}) + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := toInt64(tc.value) + if got != tc.want || ok != tc.wantOK { + t.Fatalf("toInt64(%T(%v)) = (%d, %t), want (%d, %t)", tc.value, tc.value, got, ok, tc.want, tc.wantOK) + } + }) + } +} + +func TestSummaryValueStringOversizedUnsignedDoesNotNarrow(t *testing.T) { + value := uint64(math.MaxInt64) + 1 + if got, want := summaryValueString("chunk_count", value), strconv.FormatUint(value, 10); got != want { + t.Fatalf("summaryValueString() = %q, want exact unsigned fallback %q", got, want) + } + if got := summaryValueString("chunk_count", uint64(math.MaxUint64)); strings.HasPrefix(got, "-") { + t.Fatalf("maximum uint64 rendered as negative value: %q", got) + } +} From 2776f04d6dc7ddb871371afee8c5e44e9e6fe373 Mon Sep 17 00:00:00 2001 From: franchoy Date: Thu, 20 Aug 2026 08:34:26 +0200 Subject: [PATCH 25/27] fix(security): bound inspect float conversions --- ...bservability_verify_engine_routing_test.go | 24 +++++++ internal/cli/render/inspect.go | 19 ++++-- internal/cli/render/inspect_test.go | 63 +++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/cmd/coldkeep/observability_verify_engine_routing_test.go b/cmd/coldkeep/observability_verify_engine_routing_test.go index f3efd45a..c3bfa911 100644 --- a/cmd/coldkeep/observability_verify_engine_routing_test.go +++ b/cmd/coldkeep/observability_verify_engine_routing_test.go @@ -82,6 +82,30 @@ func TestStatsAndInspectRouteThroughTypedEngineResults(t *testing.T) { } } +func TestInspectDecimalAdapterRetainsBoundedHumanProjection(t *testing.T) { + installObservabilityEngineStub(t, stubCommandEngine{ + inspectFunc: func(context.Context, engine.InspectRequest) (engine.InspectResult, error) { + return engine.InspectResult{ + Entity: engine.InspectRepository, + Summary: map[string]engine.Value{ + "compression_factor": {Kind: engine.ValueDecimal, Decimal: "2.75"}, + }, + }, nil + }, + }) + + output := captureStdout(t, func() { + if err := runInspectCommand(parsedCommandLine{ + method: "inspect", positionals: []string{"repository"}, flags: map[string][]string{}, + }, outputModeText); err != nil { + t.Fatalf("inspect: %v", err) + } + }) + if !strings.Contains(output, "compression_factor: 2") { + t.Fatalf("inspect decimal human output = %q", output) + } +} + func TestVerifyUsesOneEngineOperationAndItsSummary(t *testing.T) { dbconn := openSnapshotRoutingDB(t) originalLoad := loadDefaultStorageContextPhase diff --git a/internal/cli/render/inspect.go b/internal/cli/render/inspect.go index d4f60305..7cd54995 100644 --- a/internal/cli/render/inspect.go +++ b/internal/cli/render/inspect.go @@ -203,8 +203,8 @@ func summaryValueString(key string, value any) string { return strconv.FormatInt(n, 10) } if f, ok := toFloat64(value); ok { - if f == float64(int64(f)) { - return strconv.FormatInt(int64(f), 10) + if n, ok := truncateFiniteFloatToInt64(f); ok && f == float64(n) { + return strconv.FormatInt(n, 10) } return fmt.Sprintf("%.2f", f) } @@ -286,14 +286,25 @@ func toInt64(v any) (int64, bool) { } return int64(n), true case float64: - return int64(n), true + return truncateFiniteFloatToInt64(n) case float32: - return int64(n), true + return truncateFiniteFloatToInt64(float64(n)) default: return 0, false } } +func truncateFiniteFloatToInt64(value float64) (int64, bool) { + const ( + minInt64Inclusive = -9223372036854775808.0 + maxInt64Exclusive = 9223372036854775808.0 + ) + if math.IsNaN(value) || math.IsInf(value, 0) || value < minInt64Inclusive || value >= maxInt64Exclusive { + return 0, false + } + return int64(value), true +} + func toFloat64(v any) (float64, bool) { switch n := v.(type) { case float64: diff --git a/internal/cli/render/inspect_test.go b/internal/cli/render/inspect_test.go index 89f70322..b46c2bda 100644 --- a/internal/cli/render/inspect_test.go +++ b/internal/cli/render/inspect_test.go @@ -173,6 +173,69 @@ func TestToInt64UnsignedBoundaries(t *testing.T) { } } +func TestToInt64FloatBoundaries(t *testing.T) { + const ( + minInt64Float = -9223372036854775808.0 + maxInt64Float = 9223372036854775808.0 + ) + largestBelowMax := math.Nextafter(maxInt64Float, 0) + tests := []struct { + name string + value any + want int64 + wantOK bool + }{ + {name: "float64 fractional positive truncates", value: float64(42.75), want: 42, wantOK: true}, + {name: "float64 fractional negative truncates", value: float64(-42.75), want: -42, wantOK: true}, + {name: "float64 minimum inclusive", value: float64(minInt64Float), want: math.MinInt64, wantOK: true}, + {name: "float64 largest representable below maximum", value: largestBelowMax, want: math.MaxInt64 - 1023, wantOK: true}, + {name: "float64 maximum exclusive", value: float64(maxInt64Float), wantOK: false}, + {name: "float64 below minimum", value: math.Nextafter(minInt64Float, math.Inf(-1)), wantOK: false}, + {name: "float64 positive infinity", value: math.Inf(1), wantOK: false}, + {name: "float64 negative infinity", value: math.Inf(-1), wantOK: false}, + {name: "float64 NaN", value: math.NaN(), wantOK: false}, + {name: "float32 fractional positive truncates", value: float32(42.75), want: 42, wantOK: true}, + {name: "float32 fractional negative truncates", value: float32(-42.75), want: -42, wantOK: true}, + {name: "float32 minimum inclusive", value: float32(minInt64Float), want: math.MinInt64, wantOK: true}, + {name: "float32 below minimum", value: math.Nextafter32(float32(minInt64Float), float32(math.Inf(-1))), wantOK: false}, + {name: "float32 maximum exclusive", value: float32(maxInt64Float), wantOK: false}, + {name: "float32 positive infinity", value: float32(math.Inf(1)), wantOK: false}, + {name: "float32 NaN", value: float32(math.NaN()), wantOK: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := toInt64(tc.value) + if got != tc.want || ok != tc.wantOK { + t.Fatalf("toInt64(%T(%v)) = (%d, %t), want (%d, %t)", tc.value, tc.value, got, ok, tc.want, tc.wantOK) + } + }) + } +} + +func TestSummaryValueStringFloatSafetyAndCompatibility(t *testing.T) { + tests := []struct { + name string + key string + value any + want string + }{ + {name: "fractional human summary retains truncation", key: "compression_factor", value: 2.75, want: "2"}, + {name: "fractional byte value retains truncation", key: "size_bytes", value: 2048.75, want: "2.0 KiB"}, + {name: "positive infinity is not narrowed", key: "compression_factor", value: math.Inf(1), want: "+Inf"}, + {name: "negative infinity is not narrowed", key: "compression_factor", value: math.Inf(-1), want: "-Inf"}, + {name: "NaN is not narrowed", key: "compression_factor", value: math.NaN(), want: "NaN"}, + {name: "exclusive upper bound is not narrowed", key: "compression_factor", value: float64(9223372036854775808.0), want: "9223372036854775808.00"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := summaryValueString(tc.key, tc.value); got != tc.want { + t.Fatalf("summaryValueString(%q, %T(%v)) = %q, want %q", tc.key, tc.value, tc.value, got, tc.want) + } + }) + } +} + func TestSummaryValueStringOversizedUnsignedDoesNotNarrow(t *testing.T) { value := uint64(math.MaxInt64) + 1 if got, want := summaryValueString("chunk_count", value), strconv.FormatUint(value, 10); got != want { From f6cf721b9f27681495ce937a55480e6f3903eb0e Mon Sep 17 00:00:00 2001 From: franchoy Date: Thu, 20 Aug 2026 08:50:17 +0200 Subject: [PATCH 26/27] fix(catalog): make GC root reads coherent --- internal/catalog/gc_plan.go | 212 ++++++++--- internal/catalog/gc_plan_concurrency_test.go | 348 +++++++++++++++++++ 2 files changed, 504 insertions(+), 56 deletions(-) create mode 100644 internal/catalog/gc_plan_concurrency_test.go diff --git a/internal/catalog/gc_plan.go b/internal/catalog/gc_plan.go index ca37a317..e8eb8381 100644 --- a/internal/catalog/gc_plan.go +++ b/internal/catalog/gc_plan.go @@ -2,15 +2,75 @@ package catalog import ( "context" + "database/sql" "errors" "fmt" "sort" + "time" ) +const gcPlanMetadataQuery = ` +SELECT + CAST(0 AS BIGINT) AS row_kind, + CAST(s.id AS TEXT) AS snapshot_id, + CAST(NULL AS BIGINT) AS logical_file_id, + CAST(s.type AS TEXT) AS snapshot_type, + CAST(COALESCE(s.label, '') AS TEXT) AS snapshot_label, + CAST(COALESCE(s.parent_id, '') AS TEXT) AS snapshot_parent_id, + CAST(s.created_at AS TEXT) AS snapshot_created_at +FROM snapshot AS s +UNION ALL +SELECT + CAST(1 AS BIGINT) AS row_kind, + CAST(sf.snapshot_id AS TEXT) AS snapshot_id, + CAST(sf.logical_file_id AS BIGINT) AS logical_file_id, + CAST('' AS TEXT) AS snapshot_type, + CAST('' AS TEXT) AS snapshot_label, + CAST('' AS TEXT) AS snapshot_parent_id, + CAST('' AS TEXT) AS snapshot_created_at +FROM ( + SELECT DISTINCT snapshot_id, logical_file_id + FROM snapshot_file +) AS sf +UNION ALL +SELECT + CAST(2 AS BIGINT) AS row_kind, + CAST('' AS TEXT) AS snapshot_id, + CAST(pf.logical_file_id AS BIGINT) AS logical_file_id, + CAST('' AS TEXT) AS snapshot_type, + CAST('' AS TEXT) AS snapshot_label, + CAST('' AS TEXT) AS snapshot_parent_id, + CAST('' AS TEXT) AS snapshot_created_at +FROM ( + SELECT DISTINCT logical_file_id + FROM physical_file +) AS pf` + +const ( + gcPlanSnapshotMetadataRow int64 = iota + gcPlanSnapshotRootRow + gcPlanCurrentRootRow +) + +// gcPlanScanBarrier is an internal deterministic-concurrency test seam. The +// production path always passes nil, and no database detail crosses the public +// catalog contract. +type gcPlanScanBarrier func() + +type gcPlanSnapshotRoot struct { + snapshotID string + logicalFileID int64 +} + // LoadGCPlanMetadata implements GCPlanCatalog. // -// Phase 9 implements and adopts deterministic reachability metadata. +// All reachability inputs are read by one compound statement so PostgreSQL and +// SQLite cannot expose metadata and roots from different statement snapshots. func (s *Service) LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*GCPlanMetadata, error) { + return s.loadGCPlanMetadata(ctx, input, nil) +} + +func (s *Service) loadGCPlanMetadata(ctx context.Context, input GCPlanInput, barrier gcPlanScanBarrier) (*GCPlanMetadata, error) { if ctx == nil { ctx = context.Background() } @@ -21,23 +81,87 @@ func (s *Service) LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*G if err != nil { return nil, err } + + rows, err := s.db.QueryContext(ctx, gcPlanMetadataQuery) + if err != nil { + return nil, gcCatalogError(err) + } + defer func() { _ = rows.Close() }() + + snapshots := make([]SnapshotRef, 0) + snapshotIDs := make(map[string]struct{}) + snapshotRoots := make([]gcPlanSnapshotRoot, 0) + currentLogicalIDs := make([]int64, 0) + rowNumber := 0 + for rows.Next() { + var ( + rowKind int64 + snapshotID sql.NullString + logicalFileID sql.NullInt64 + snapshotType sql.NullString + label sql.NullString + parentID sql.NullString + createdAt sql.NullString + ) + if err := rows.Scan(&rowKind, &snapshotID, &logicalFileID, &snapshotType, &label, &parentID, &createdAt); err != nil { + return nil, gcCatalogError(fmt.Errorf("scan GC metadata row: %w", err)) + } + rowNumber++ + if rowNumber == 1 && barrier != nil { + barrier() + } + + switch rowKind { + case gcPlanSnapshotMetadataRow: + if !snapshotID.Valid || snapshotID.String == "" || !snapshotType.Valid || !createdAt.Valid { + return nil, NewError(ErrorInvariantViolation, "load GC plan", "snapshot_metadata_complete", "snapshot metadata row is incomplete", nil) + } + if _, duplicate := snapshotIDs[snapshotID.String]; duplicate { + return nil, NewError(ErrorInvariantViolation, "load GC plan", "snapshot_metadata_unique", fmt.Sprintf("snapshot metadata %q is duplicated", snapshotID.String), nil) + } + snapshotIDs[snapshotID.String] = struct{}{} + snapshots = append(snapshots, SnapshotRef{ + ID: snapshotID.String, + Type: snapshotType.String, + Label: label.String, + ParentID: parentID.String, + CreatedAt: parseGCTimestamp(createdAt.String), + }) + case gcPlanSnapshotRootRow: + if !snapshotID.Valid || snapshotID.String == "" || !logicalFileID.Valid { + return nil, NewError(ErrorInvariantViolation, "load GC plan", "snapshot_root_complete", "snapshot root row is incomplete", nil) + } + snapshotRoots = append(snapshotRoots, gcPlanSnapshotRoot{snapshotID: snapshotID.String, logicalFileID: logicalFileID.Int64}) + case gcPlanCurrentRootRow: + if !logicalFileID.Valid { + return nil, NewError(ErrorInvariantViolation, "load GC plan", "current_root_complete", "current root row is incomplete", nil) + } + currentLogicalIDs = append(currentLogicalIDs, logicalFileID.Int64) + default: + return nil, NewError(ErrorInvariantViolation, "load GC plan", "metadata_row_kind", fmt.Sprintf("unknown GC metadata row kind %d", rowKind), nil) + } + } + if err := rows.Err(); err != nil { + return nil, gcCatalogError(fmt.Errorf("iterate GC metadata rows: %w", err)) + } + if err := rows.Close(); err != nil { + return nil, gcCatalogError(fmt.Errorf("close GC metadata rows: %w", err)) + } + excluded := make(map[string]struct{}, len(normalized.ExcludeSnapshotIDs)) for _, id := range normalized.ExcludeSnapshotIDs { - var exists bool - if err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM snapshot WHERE id = $1)`, id).Scan(&exists); err != nil { - return nil, gcCatalogError(fmt.Errorf("validate excluded snapshot %q: %w", id, err)) - } - if !exists { + if _, exists := snapshotIDs[id]; !exists { return nil, NewError(ErrorNotFound, "load GC plan", "excluded_snapshot_exists", fmt.Sprintf("snapshot %q does not exist", id), nil) } excluded[id] = struct{}{} } - - snapshots, err := s.ListSnapshots(ctx, SnapshotFilter{}) - if err != nil { - return nil, gcCatalogError(fmt.Errorf("load protected snapshots: %w", err)) + for _, root := range snapshotRoots { + if _, exists := snapshotIDs[root.snapshotID]; !exists { + return nil, NewError(ErrorInvariantViolation, "load GC plan", "snapshot_root_has_snapshot", fmt.Sprintf("snapshot root %q has no protected snapshot metadata", root.snapshotID), nil) + } } - protected := make([]SnapshotRef, 0, len(snapshots)) + + protected := make([]SnapshotRef, 0, len(snapshots)-len(excluded)) for _, snapshot := range snapshots { if _, skip := excluded[snapshot.ID]; !skip { protected = append(protected, snapshot) @@ -55,54 +179,15 @@ func (s *Service) LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*G } roots := make(map[int64]*GCReachabilityRoot) - currentRows, err := s.db.QueryContext(ctx, `SELECT DISTINCT logical_file_id FROM physical_file ORDER BY logical_file_id`) - if err != nil { - return nil, gcCatalogError(fmt.Errorf("load current GC roots: %w", err)) - } - for currentRows.Next() { - var id int64 - if err := currentRows.Scan(&id); err != nil { - _ = currentRows.Close() - return nil, gcCatalogError(fmt.Errorf("scan current GC root: %w", err)) - } - root := ensureGCRoot(roots, id) - root.Current = true - } - if err := currentRows.Err(); err != nil { - _ = currentRows.Close() - return nil, gcCatalogError(fmt.Errorf("iterate current GC roots: %w", err)) - } - if err := currentRows.Close(); err != nil { - return nil, gcCatalogError(fmt.Errorf("close current GC roots: %w", err)) + for _, logicalID := range currentLogicalIDs { + ensureGCRoot(roots, logicalID).Current = true } - - snapshotRows, err := s.db.QueryContext(ctx, `SELECT DISTINCT snapshot_id, logical_file_id FROM snapshot_file ORDER BY logical_file_id, snapshot_id`) - if err != nil { - return nil, gcCatalogError(fmt.Errorf("load snapshot GC roots: %w", err)) - } - for snapshotRows.Next() { - var snapshotID string - var logicalID int64 - if err := snapshotRows.Scan(&snapshotID, &logicalID); err != nil { - _ = snapshotRows.Close() - return nil, gcCatalogError(fmt.Errorf("scan snapshot GC root: %w", err)) - } - if _, skip := excluded[snapshotID]; skip { + for _, snapshotRoot := range snapshotRoots { + if _, skip := excluded[snapshotRoot.snapshotID]; skip { continue } - if _, known := snapshotRank[snapshotID]; !known { - _ = snapshotRows.Close() - return nil, NewError(ErrorInvariantViolation, "load GC plan", "snapshot_root_has_snapshot", fmt.Sprintf("snapshot root %q has no protected snapshot metadata", snapshotID), nil) - } - root := ensureGCRoot(roots, logicalID) - root.SnapshotIDs = append(root.SnapshotIDs, snapshotID) - } - if err := snapshotRows.Err(); err != nil { - _ = snapshotRows.Close() - return nil, gcCatalogError(fmt.Errorf("iterate snapshot GC roots: %w", err)) - } - if err := snapshotRows.Close(); err != nil { - return nil, gcCatalogError(fmt.Errorf("close snapshot GC roots: %w", err)) + root := ensureGCRoot(roots, snapshotRoot.logicalFileID) + root.SnapshotIDs = append(root.SnapshotIDs, snapshotRoot.snapshotID) } logicalIDs := make([]int64, 0, len(roots)) @@ -121,6 +206,21 @@ func (s *Service) LoadGCPlanMetadata(ctx context.Context, input GCPlanInput) (*G return &GCPlanMetadata{Roots: orderedRoots, ProtectedSnapshots: protected}, nil } +func parseGCTimestamp(value string) time.Time { + if parsed := parseTimestamp(value); !parsed.IsZero() { + return parsed + } + for _, layout := range []string{ + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05.999999999Z07", + } { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed + } + } + return time.Time{} +} + func ensureGCRoot(roots map[int64]*GCReachabilityRoot, id int64) *GCReachabilityRoot { if root := roots[id]; root != nil { return root diff --git a/internal/catalog/gc_plan_concurrency_test.go b/internal/catalog/gc_plan_concurrency_test.go new file mode 100644 index 00000000..7b2de33a --- /dev/null +++ b/internal/catalog/gc_plan_concurrency_test.go @@ -0,0 +1,348 @@ +package catalog + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + "time" + + "github.com/franchoy/coldkeep/internal/db" + "github.com/franchoy/coldkeep/internal/testutil/backendtest" + _ "github.com/mattn/go-sqlite3" +) + +type gcPlanRaceResult struct { + plan *GCPlanMetadata + err error +} + +type gcPlanRaceCase struct { + name string + input GCPlanInput + mutate func(context.Context, *sql.Tx) error + assertDuring func(*testing.T, *GCPlanMetadata, error) + assertAfter func(*testing.T, *GCPlanMetadata, error) +} + +func TestGCPlanCompoundReadIsStatementCoherentAcrossBackends(t *testing.T) { + cases := []gcPlanRaceCase{ + { + name: "snapshot creation", + mutate: func(ctx context.Context, tx *sql.Tx) error { + pathID, err := insertGCRaceSnapshotPath(ctx, tx, "/race/create") + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "INSERT INTO snapshot (id, created_at, type, label) VALUES ($1, $2, $3, $4)", + "snap-create", time.Date(2026, 1, 4, 0, 0, 0, 0, time.UTC), "full", "created"); err != nil { + return err + } + _, err = tx.ExecContext(ctx, "INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id) VALUES ($1, $2, $3)", + "snap-create", pathID, int64(101)) + return err + }, + assertDuring: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceSnapshot(t, plan, "snap-create", false) + assertGCRaceRootSnapshot(t, plan, 101, "snap-create", false) + }, + assertAfter: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceSnapshot(t, plan, "snap-create", true) + assertGCRaceRootSnapshot(t, plan, 101, "snap-create", true) + }, + }, + { + name: "snapshot deletion", + mutate: func(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, "DELETE FROM snapshot_file WHERE snapshot_id = $1", "snap-delete"); err != nil { + return err + } + _, err := tx.ExecContext(ctx, "DELETE FROM snapshot WHERE id = $1", "snap-delete") + return err + }, + assertDuring: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceSnapshot(t, plan, "snap-delete", true) + assertGCRaceRootSnapshot(t, plan, 101, "snap-delete", true) + }, + assertAfter: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceSnapshot(t, plan, "snap-delete", false) + assertGCRaceRootSnapshot(t, plan, 101, "snap-delete", false) + }, + }, + { + name: "excluded snapshot deletion", + input: GCPlanInput{ExcludeSnapshotIDs: []string{"snap-excluded"}}, + mutate: func(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, "DELETE FROM snapshot_file WHERE snapshot_id = $1", "snap-excluded"); err != nil { + return err + } + _, err := tx.ExecContext(ctx, "DELETE FROM snapshot WHERE id = $1", "snap-excluded") + return err + }, + assertDuring: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceSnapshot(t, plan, "snap-excluded", false) + assertGCRaceRootSnapshot(t, plan, 101, "snap-excluded", false) + }, + assertAfter: func(t *testing.T, plan *GCPlanMetadata, err error) { + t.Helper() + if plan != nil || !IsCode(err, ErrorNotFound) { + t.Fatalf("post-delete excluded plan = %+v, err = %v; want not_found", plan, err) + } + }, + }, + { + name: "current root insertion", + mutate: func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, "INSERT INTO physical_file (path, logical_file_id) VALUES ($1, $2)", "/current/insert", int64(101)) + return err + }, + assertDuring: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceCurrentRoot(t, plan, 101, false) + }, + assertAfter: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceCurrentRoot(t, plan, 101, true) + }, + }, + { + name: "current root removal", + mutate: func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, "DELETE FROM physical_file WHERE path = $1", "/current/remove") + return err + }, + assertDuring: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceCurrentRoot(t, plan, 102, true) + }, + assertAfter: func(t *testing.T, plan *GCPlanMetadata, err error) { + plan = requireGCRacePlan(t, plan, err) + assertGCRaceCurrentRoot(t, plan, 102, false) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + backendtest.ForEach(t, backendtest.Options{Postgres: backendtest.PostgresOptional}, func(t *testing.T, backend backendtest.Backend) { + prepareGCRaceBackend(t, backend) + seedGCRaceFixture(t, backend.DB) + runGCRaceInterleaving(t, backend.DB, tc) + }) + }) + } +} + +func prepareGCRaceBackend(t *testing.T, backend backendtest.Backend) { + t.Helper() + backend.DB.SetMaxOpenConns(4) + if backend.Kind != db.BackendSQLite { + return + } + var mode string + if err := backend.DB.QueryRow("PRAGMA journal_mode = WAL").Scan(&mode); err != nil { + t.Fatalf("enable SQLite WAL: %v", err) + } + if strings.ToLower(strings.TrimSpace(mode)) != "wal" { + t.Fatalf("enable SQLite WAL: journal_mode=%q, want wal", mode) + } + if err := backend.DB.QueryRow("PRAGMA journal_mode").Scan(&mode); err != nil { + t.Fatalf("verify SQLite WAL: %v", err) + } + if strings.ToLower(strings.TrimSpace(mode)) != "wal" { + t.Fatalf("verify SQLite WAL: journal_mode=%q, want wal", mode) + } +} + +func seedGCRaceFixture(t *testing.T, dbconn *sql.DB) { + t.Helper() + ctx := context.Background() + for _, logicalID := range []int64{101, 102} { + if _, err := dbconn.ExecContext(ctx, + "INSERT INTO logical_file (id, original_name, total_size, file_hash, ref_count, status) VALUES ($1, $2, $3, $4, $5, $6)", + logicalID, fmt.Sprintf("race-%d", logicalID), logicalID, fmt.Sprintf("race-hash-%d", logicalID), 1, "COMPLETED"); err != nil { + t.Fatalf("seed logical file %d: %v", logicalID, err) + } + } + seedGCRaceSnapshot(t, dbconn, "snap-zero", "/race/zero", 0, time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + seedGCRaceSnapshot(t, dbconn, "snap-delete", "/race/delete", 101, time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)) + seedGCRaceSnapshot(t, dbconn, "snap-excluded", "/race/excluded", 101, time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) + if _, err := dbconn.ExecContext(ctx, "INSERT INTO physical_file (path, logical_file_id) VALUES ($1, $2)", "/current/remove", int64(102)); err != nil { + t.Fatalf("seed current root: %v", err) + } +} + +func seedGCRaceSnapshot(t *testing.T, dbconn *sql.DB, snapshotID, path string, logicalID int64, createdAt time.Time) { + t.Helper() + ctx := context.Background() + if _, err := dbconn.ExecContext(ctx, "INSERT INTO snapshot (id, created_at, type, label) VALUES ($1, $2, $3, $4)", + snapshotID, createdAt, "full", snapshotID); err != nil { + t.Fatalf("seed snapshot %q: %v", snapshotID, err) + } + if logicalID == 0 { + return + } + pathID, err := insertGCRaceSnapshotPath(ctx, dbconn, path) + if err != nil { + t.Fatalf("seed snapshot path %q: %v", path, err) + } + if _, err := dbconn.ExecContext(ctx, "INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id) VALUES ($1, $2, $3)", + snapshotID, pathID, logicalID); err != nil { + t.Fatalf("seed snapshot root %q: %v", snapshotID, err) + } +} + +type gcPlanRaceDB interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func insertGCRaceSnapshotPath(ctx context.Context, dbconn gcPlanRaceDB, path string) (int64, error) { + if _, err := dbconn.ExecContext(ctx, "INSERT INTO snapshot_path (path) VALUES ($1)", path); err != nil { + return 0, err + } + var pathID int64 + if err := dbconn.QueryRowContext(ctx, "SELECT id FROM snapshot_path WHERE path = $1", path).Scan(&pathID); err != nil { + return 0, err + } + return pathID, nil +} + +func runGCRaceInterleaving(t *testing.T, dbconn *sql.DB, tc gcPlanRaceCase) { + t.Helper() + reached := make(chan struct{}) + resume := make(chan struct{}) + resultCh := make(chan gcPlanRaceResult, 1) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + svc := NewServiceFromSQL(dbconn) + go func() { + plan, err := svc.loadGCPlanMetadata(ctx, tc.input, func() { + close(reached) + <-resume + }) + resultCh <- gcPlanRaceResult{plan: plan, err: err} + }() + + select { + case <-reached: + case <-ctx.Done(): + t.Fatalf("GC read did not reach scan barrier: %v", ctx.Err()) + } + + tx, err := dbconn.BeginTx(ctx, nil) + if err != nil { + close(resume) + t.Fatalf("begin concurrent mutation: %v", err) + } + if err := tc.mutate(ctx, tx); err != nil { + _ = tx.Rollback() + close(resume) + t.Fatalf("execute concurrent mutation: %v", err) + } + if err := tx.Commit(); err != nil { + close(resume) + t.Fatalf("commit concurrent mutation: %v", err) + } + close(resume) + + var during gcPlanRaceResult + select { + case during = <-resultCh: + case <-ctx.Done(): + t.Fatalf("GC read did not complete after barrier release: %v", ctx.Err()) + } + tc.assertDuring(t, during.plan, during.err) + + after, afterErr := svc.LoadGCPlanMetadata(context.Background(), tc.input) + tc.assertAfter(t, after, afterErr) +} + +func requireGCRacePlan(t *testing.T, plan *GCPlanMetadata, err error) *GCPlanMetadata { + t.Helper() + if err != nil || plan == nil { + t.Fatalf("GC plan = %+v, err = %v", plan, err) + } + assertGCRaceSnapshot(t, plan, "snap-zero", true) + return plan +} + +func assertGCRaceSnapshot(t *testing.T, plan *GCPlanMetadata, snapshotID string, want bool) { + t.Helper() + found := false + for _, snapshot := range plan.ProtectedSnapshots { + if snapshot.ID == snapshotID { + found = true + break + } + } + if found != want { + t.Fatalf("protected snapshot %q present=%t, want %t: %+v", snapshotID, found, want, plan.ProtectedSnapshots) + } +} + +func assertGCRaceRootSnapshot(t *testing.T, plan *GCPlanMetadata, logicalID int64, snapshotID string, want bool) { + t.Helper() + found := false + if root := findGCRaceRoot(plan, logicalID); root != nil { + for _, id := range root.SnapshotIDs { + if id == snapshotID { + found = true + break + } + } + } + if found != want { + t.Fatalf("root %d snapshot %q present=%t, want %t: %+v", logicalID, snapshotID, found, want, plan.Roots) + } +} + +func assertGCRaceCurrentRoot(t *testing.T, plan *GCPlanMetadata, logicalID int64, want bool) { + t.Helper() + root := findGCRaceRoot(plan, logicalID) + found := root != nil && root.Current + if found != want { + t.Fatalf("root %d current=%t, want %t: %+v", logicalID, found, want, plan.Roots) + } +} + +func findGCRaceRoot(plan *GCPlanMetadata, logicalID int64) *GCReachabilityRoot { + for i := range plan.Roots { + if plan.Roots[i].LogicalFileID == logicalID { + return &plan.Roots[i] + } + } + return nil +} + +func TestGCPlanCompoundReadPreservesOrphanDetection(t *testing.T) { + path := t.TempDir() + "/orphan.sqlite" + dbconn, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("open SQLite: %v", err) + } + t.Cleanup(func() { _ = dbconn.Close() }) + dbconn.SetMaxOpenConns(1) + if err := db.EnsureSchema(dbconn); err != nil { + t.Fatalf("bootstrap SQLite: %v", err) + } + if _, err := dbconn.Exec("PRAGMA foreign_keys = OFF"); err != nil { + t.Fatalf("disable fixture foreign keys: %v", err) + } + if _, err := dbconn.Exec("INSERT INTO snapshot_file (snapshot_id, path_id, logical_file_id) VALUES ($1, $2, $3)", + "orphan-snapshot", int64(999), int64(999)); err != nil { + t.Fatalf("insert orphan fixture: %v", err) + } + + plan, err := NewServiceFromSQL(dbconn).LoadGCPlanMetadata(context.Background(), GCPlanInput{}) + if plan != nil || !IsCode(err, ErrorInvariantViolation) || !strings.Contains(err.Error(), "orphan-snapshot") { + t.Fatalf("orphan plan = %+v, err = %v; want invariant_violation naming orphan", plan, err) + } +} From 33aa1a563b1e6f7b09a86326c6bbd06d7b106e58 Mon Sep 17 00:00:00 2001 From: franchoy Date: Thu, 20 Aug 2026 10:10:54 +0200 Subject: [PATCH 27/27] fix(security): preserve exact unsigned inspect integers --- cmd/coldkeep/observability_engine_adapters.go | 4 +- .../observability_engine_adapters_test.go | 97 ++++++++++++++++++- ...bservability_verify_engine_routing_test.go | 76 +++++++++++++++ 3 files changed, 171 insertions(+), 6 deletions(-) diff --git a/cmd/coldkeep/observability_engine_adapters.go b/cmd/coldkeep/observability_engine_adapters.go index b837d82d..17bc94ec 100644 --- a/cmd/coldkeep/observability_engine_adapters.go +++ b/cmd/coldkeep/observability_engine_adapters.go @@ -182,8 +182,8 @@ func engineValueToAny(input engine.Value) (any, error) { if value, err := strconv.ParseInt(input.Integer, 10, 64); err == nil { return value, nil } - if value, err := strconv.ParseUint(input.Integer, 10, 64); err == nil { - return value, nil + if _, err := strconv.ParseUint(input.Integer, 10, 64); err == nil { + return json.Number(input.Integer), nil } if _, err := json.Number(input.Integer).Float64(); err != nil { return nil, fmt.Errorf("invalid integer %q", input.Integer) diff --git a/cmd/coldkeep/observability_engine_adapters_test.go b/cmd/coldkeep/observability_engine_adapters_test.go index 7b6293aa..d7bb5601 100644 --- a/cmd/coldkeep/observability_engine_adapters_test.go +++ b/cmd/coldkeep/observability_engine_adapters_test.go @@ -1,23 +1,112 @@ package main import ( + "bytes" "encoding/json" + "fmt" + "reflect" "testing" "github.com/franchoy/coldkeep/internal/engine" ) -func TestEngineValueToAnyPreservesLargeIntegerJSONToken(t *testing.T) { - value, err := engineValueToAny(engine.Value{Kind: engine.ValueInteger, Integer: "18446744073709551615"}) +func TestEngineValueToAnyPreservesIntegerBoundaries(t *testing.T) { + tests := []struct { + name string + integer string + want any + }{ + {name: "zero", integer: "0", want: int64(0)}, + {name: "maximum int64", integer: "9223372036854775807", want: int64(9223372036854775807)}, + {name: "one above maximum int64", integer: "9223372036854775808", want: json.Number("9223372036854775808")}, + {name: "maximum uint64", integer: "18446744073709551615", want: json.Number("18446744073709551615")}, + {name: "one above maximum uint64", integer: "18446744073709551616", want: json.Number("18446744073709551616")}, + {name: "minimum int64", integer: "-9223372036854775808", want: int64(-9223372036854775808)}, + {name: "one below minimum int64", integer: "-9223372036854775809", want: json.Number("-9223372036854775809")}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assertEngineIntegerProjection(t, tc.integer, tc.want) + }) + } +} + +func assertEngineIntegerProjection(t *testing.T, integer string, want any) { + t.Helper() + value, err := engineValueToAny(engine.Value{Kind: engine.ValueInteger, Integer: integer}) + if err != nil { + t.Fatalf("engineValueToAny: %v", err) + } + if !reflect.DeepEqual(value, want) { + t.Fatalf("engineValueToAny type/value = %T(%v), want %T(%v)", value, value, want, want) + } + if got := fmt.Sprint(value); got != integer { + t.Fatalf("human integer token changed: got=%q want=%q", got, integer) + } + + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + if got := string(encoded); got != integer { + t.Fatalf("JSON integer token changed: got=%s want=%s", got, integer) + } + + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + t.Fatalf("decode JSON integer token: %v", err) + } + number, ok := decoded.(json.Number) + if !ok || string(number) != integer { + t.Fatalf("decoded JSON value = %T(%v), want json.Number(%q)", decoded, decoded, integer) + } +} + +func TestEngineValueToAnyPreservesNestedIntegerBoundaries(t *testing.T) { + value, err := engineValueToAny(engine.Value{ + Kind: engine.ValueObject, + Object: map[string]engine.Value{ + "array": { + Kind: engine.ValueArray, + Array: []engine.Value{ + {Kind: engine.ValueInteger, Integer: "0"}, + {Kind: engine.ValueInteger, Integer: "9223372036854775808"}, + {Kind: engine.ValueInteger, Integer: "18446744073709551615"}, + {Kind: engine.ValueInteger, Integer: "18446744073709551616"}, + { + Kind: engine.ValueObject, + Object: map[string]engine.Value{ + "below": {Kind: engine.ValueInteger, Integer: "-9223372036854775809"}, + }, + }, + }, + }, + }, + }) if err != nil { t.Fatalf("engineValueToAny: %v", err) } + want := map[string]any{ + "array": []any{ + int64(0), + json.Number("9223372036854775808"), + json.Number("18446744073709551615"), + json.Number("18446744073709551616"), + map[string]any{"below": json.Number("-9223372036854775809")}, + }, + } + if !reflect.DeepEqual(value, want) { + t.Fatalf("nested projection = %#v, want %#v", value, want) + } encoded, err := json.Marshal(value) if err != nil { t.Fatalf("json.Marshal: %v", err) } - if got, want := string(encoded), "18446744073709551615"; got != want { - t.Fatalf("integer token changed: got=%s want=%s", got, want) + if got, wantJSON := string(encoded), `{"array":[0,9223372036854775808,18446744073709551615,18446744073709551616,{"below":-9223372036854775809}]}`; got != wantJSON { + t.Fatalf("nested JSON integer tokens changed: got=%s want=%s", got, wantJSON) } } diff --git a/cmd/coldkeep/observability_verify_engine_routing_test.go b/cmd/coldkeep/observability_verify_engine_routing_test.go index c3bfa911..81888914 100644 --- a/cmd/coldkeep/observability_verify_engine_routing_test.go +++ b/cmd/coldkeep/observability_verify_engine_routing_test.go @@ -106,6 +106,82 @@ func TestInspectDecimalAdapterRetainsBoundedHumanProjection(t *testing.T) { } } +func inspectIntegerBoundaryTokens() map[string]string { + return map[string]string{ + "above_max_int64": "9223372036854775808", + "max_uint64": "18446744073709551615", + "above_max_uint64": "18446744073709551616", + "below_min_int64": "-9223372036854775809", + } +} + +func installInspectIntegerBoundaryStub(t *testing.T) { + integerSummary := map[string]engine.Value{ + "above_max_int64": {Kind: engine.ValueInteger, Integer: "9223372036854775808"}, + "max_uint64": {Kind: engine.ValueInteger, Integer: "18446744073709551615"}, + "above_max_uint64": {Kind: engine.ValueInteger, Integer: "18446744073709551616"}, + "below_min_int64": {Kind: engine.ValueInteger, Integer: "-9223372036854775809"}, + } + installObservabilityEngineStub(t, stubCommandEngine{ + inspectFunc: func(context.Context, engine.InspectRequest) (engine.InspectResult, error) { + return engine.InspectResult{Entity: engine.InspectRepository, Summary: integerSummary}, nil + }, + }) +} + +func TestInspectIntegerAdapterPreservesLargeHumanNumbers(t *testing.T) { + installInspectIntegerBoundaryStub(t) + humanOutput := captureStdout(t, func() { + if err := runInspectCommand(parsedCommandLine{ + method: "inspect", positionals: []string{"repository"}, flags: map[string][]string{}, + }, outputModeText); err != nil { + t.Fatalf("inspect human: %v", err) + } + }) + for key, value := range inspectIntegerBoundaryTokens() { + if want := key + ": " + value; !strings.Contains(humanOutput, want) { + t.Fatalf("inspect human output missing %q: %s", want, humanOutput) + } + } +} + +func TestInspectIntegerAdapterPreservesLargeJSONNumbers(t *testing.T) { + installInspectIntegerBoundaryStub(t) + jsonOutput := captureStdout(t, func() { + if err := runInspectCommand(parsedCommandLine{ + method: "inspect", positionals: []string{"repository"}, flags: map[string][]string{"output": {"json"}}, + }, outputModeJSON); err != nil { + t.Fatalf("inspect JSON: %v", err) + } + }) + summary := decodeInspectJSONSummary(t, jsonOutput) + for key, want := range inspectIntegerBoundaryTokens() { + got, ok := summary[key].(json.Number) + if !ok || string(got) != want { + t.Fatalf("inspect JSON summary[%q] = %T(%v), want json.Number(%q)", key, summary[key], summary[key], want) + } + } +} + +func decodeInspectJSONSummary(t *testing.T, output string) map[string]any { + t.Helper() + decoder := json.NewDecoder(strings.NewReader(output)) + decoder.UseNumber() + var payload map[string]any + if err := decoder.Decode(&payload); err != nil { + t.Fatalf("decode inspect JSON: %v\n%s", err, output) + } + data, ok := payload["data"].(map[string]any) + if !ok { + t.Fatalf("inspect JSON data = %T(%v)", payload["data"], payload["data"]) + } + summary, ok := data["summary"].(map[string]any) + if !ok { + t.Fatalf("inspect JSON summary = %T(%v)", data["summary"], data["summary"]) + } + return summary +} + func TestVerifyUsesOneEngineOperationAndItsSummary(t *testing.T) { dbconn := openSnapshotRoutingDB(t) originalLoad := loadDefaultStorageContextPhase